diff --git a/CMakeLists.txt b/CMakeLists.txt index 16a59d90a..42a4fce86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.1) # to download dependencies if(NOT UNIX) - message(FATAL "Unsupported operating system.") + message(FATAL_ERROR "Unsupported operating system.") endif() # Set `make clean` to ignore outputs of add_custom_command. If generated files @@ -171,17 +171,26 @@ add_library(antlr_opencypher_parser_lib STATIC ${antlr_opencypher_generated_src} target_link_libraries(antlr_opencypher_parser_lib antlr4) # ----------------------------------------------------------------------------- -include_directories(src) -add_subdirectory(src) -# ----------------------------------------------------------------------------- - # Optional subproject configuration ------------------------------------------- option(POC "Build proof of concept binaries" ON) option(EXPERIMENTAL "Build experimental binaries" OFF) option(CUSTOMERS "Build customer binaries" ON) -option(TEST_COVERAGE "Generate coverage reports from unit tests" OFF) +option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF) option(TOOLS "Build tools binaries" OFF) +if (TEST_COVERAGE) + string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) + if (NOT lower_build_type STREQUAL "debug") + message(FATAL_ERROR "Generating test coverage unsupported in non Debug builds. Current build type is '${CMAKE_BUILD_TYPE}'") + endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping") +endif() + +# Add subprojects +include_directories(src) +add_subdirectory(src) + if(POC) add_subdirectory(poc) endif() diff --git a/apollo_build.yaml b/apollo_build.yaml index 301b7d91d..20e8bcbff 100644 --- a/apollo_build.yaml +++ b/apollo_build.yaml @@ -9,9 +9,16 @@ doxygen Doxyfile cd build - cmake -DTOOLS=ON -DTEST_COVERAGE=ON .. + cmake -DTOOLS=ON .. TIMEOUT=1000 make -j$THREADS + cd .. + mkdir build_coverage + + cd build_coverage + cmake -DTEST_COVERAGE=ON .. + TIMEOUT=1000 make -j$THREADS memgraph__unit + cd .. mkdir build_release diff --git a/coverage b/coverage deleted file mode 100755 index f2aeef437..000000000 --- a/coverage +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# coverage applies only on unit tests -unit_test_dir='build/tests/unit' -coverage_dir='build/coverage' -coverage_file='coverage.info' - -# execute unit tests to generate coverage files -pushd ${unit_test_dir} -ctest -R unit --quiet -popd - -# generate coverage info files for each unit test binary -# + exclude stl, tests and libs -cd ${working_dir} -binary_path="${unit_test_dir}/CMakeFiles" -all_coverage_info="" -for dir in $binary_path/*.dir; do - pushd ${dir} - lcov --gcov-tool ${working_dir}/llvm-gcov -c -d . -o ${coverage_file} - lcov -r ${coverage_file} '/usr/*' '*/libs/*' '*/tests/*' -o ${coverage_file} - test_coverage_file=${working_dir}/${dir}/${coverage_file} - if [ -f ${coverage_file} ]; then - if [ -s ${coverage_file} ]; then - char_no=`wc -c ${coverage_file} | awk '{print $1}'` - if [ "${char_no}" -eq "0" ]; then - echo "PASS: ${test_coverage_file} contains 0 chars." - popd - continue - fi - else - echo "PASS: ${test_coverage_file} is empty." - popd - continue - fi - else - echo "PASS: ${test_coverage_file} doesn't exist." - popd - continue - fi - all_coverage_info+=" -a ${working_dir}/${dir}/${coverage_file}" - popd -done - -# merge all info files into total coverage info and generate html -echo ${all_coverage_info} -cd ${working_dir} -mkdir -p ${coverage_dir} -pushd ${coverage_dir} -lcov ${all_coverage_info} -o ${coverage_file} -genhtml ${coverage_file} -popd - -# generage coberatura xml file (compatible with Jenkins) -cd ${working_dir} -python libs/lcov-to-cobertura-xml/lcov_cobertura/lcov_cobertura.py \ - ${coverage_dir}/${coverage_file} -o ${coverage_dir}/coverage.xml diff --git a/experimental/distributed/tests/CMakeLists.txt b/experimental/distributed/tests/CMakeLists.txt index 76bd73033..335e51476 100644 --- a/experimental/distributed/tests/CMakeLists.txt +++ b/experimental/distributed/tests/CMakeLists.txt @@ -17,10 +17,6 @@ foreach(test_cpp ${test_type_cpps}) # build exec file add_executable(${target_name} ${test_cpp}) - if(${TEST_COVERAGE}) - set_target_properties(${target_name} PROPERTIES COMPILE_FLAGS "-g -O0 -Wall -fprofile-arcs -ftest-coverage") - set_target_properties(${target_name} PROPERTIES LINK_FLAGS "--coverage -fprofile-arcs -ftest-coverage") - endif() # OUTPUT_NAME sets the real name of a target when it is built and can be # used to help create two targets of the same name even though CMake @@ -31,10 +27,6 @@ foreach(test_cpp ${test_type_cpps}) target_link_libraries(${target_name} distributed_lib memgraph_lib) # gtest target_link_libraries(${target_name} gtest gtest_main) - if(${TEST_COVERAGE}) - # for code coverage - target_link_libraries(${target_name} gcov) - endif() # register test set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml) diff --git a/libs/setup.sh b/libs/setup.sh index b53b4d8a1..b1f59a6e6 100755 --- a/libs/setup.sh +++ b/libs/setup.sh @@ -86,11 +86,6 @@ clone git://deps.memgraph.io/googletest.git googletest $googletest_tag glog_tag="042a21657e79784226babab8b942f7bd0949635f" # custom version (v0.3.5+) clone git://deps.memgraph.io/glog.git glog $glog_tag -# lcov-to-coberatura-xml -# git clone https://github.com/eriwen/lcov-to-cobertura-xml.git -lcov_to_xml_tag="59584761cb5da4687693faec05bf3e2b74e9dde9" # Dec 6, 2016 -clone git://deps.memgraph.io/lcov-to-cobertura-xml.git lcov-to-cobertura-xml $lcov_to_xml_tag - # google flags # git clone https://github.com/memgraph/gflags.git gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017) diff --git a/llvm-gcov b/llvm-gcov deleted file mode 100755 index 20665e156..000000000 --- a/llvm-gcov +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -exec llvm-cov-3.8 gcov "$@" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index aa53a46bc..ad68fffe2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -51,6 +51,8 @@ set(memgraph_src_files ) # ----------------------------------------------------------------------------- +string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) + # memgraph_lib depend on these libraries set(MEMGRAPH_ALL_LIBS stdc++fs Threads::Threads fmt cppitertools antlr_opencypher_parser_lib dl glog gflags ${Boost_SERIALIZATION_LIBRARY_RELEASE}) @@ -90,8 +92,6 @@ add_custom_command(TARGET memgraph POST_BUILD COMMENT Creating symlink to memgraph executable) # Strip the executable in release build. -string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) - if (lower_build_type STREQUAL "release") add_custom_command(TARGET memgraph POST_BUILD COMMAND strip -s $ diff --git a/tests/apollo_runs.py b/tests/apollo_runs.py index 3374f4a59..c458dad7b 100755 --- a/tests/apollo_runs.py +++ b/tests/apollo_runs.py @@ -38,9 +38,7 @@ runs = [] for test in tests: order, name, path = test dirname, basename = os.path.split(path) - cmakedir = os.path.join("CMakeFiles", - "memgraph" + CTEST_DELIMITER + name + ".dir") - files = [basename, cmakedir] + files = [basename] # extra files for specific tests if name == "unit__fswatcher": @@ -56,10 +54,11 @@ for test in tests: prefix = "TIMEOUT=600 " outfile_paths = [] - if name.startswith("unit"): - cmakedir_abs = os.path.join(TESTS_DIR, "unit", cmakedir) - cmakedir_rel = os.path.relpath(cmakedir_abs, WORKSPACE_DIR) - outfile_paths.append("\./" + cmakedir_rel.replace(".", "\\.") + ".+") + if name.startswith("unit") and mode == "diff": + dirname = dirname.replace("/build/", "/build_coverage/") + curdir_abs = os.path.normpath(os.path.join(SCRIPT_DIR, dirname)) + curdir_rel = os.path.relpath(curdir_abs, WORKSPACE_DIR) + outfile_paths.append("\./" + curdir_rel.replace(".", "\\.") + "/.+") runs.append({ "name": name, diff --git a/tests/distributed/raft/CMakeLists.txt b/tests/distributed/raft/CMakeLists.txt index 3c2e0a27e..c3624024a 100644 --- a/tests/distributed/raft/CMakeLists.txt +++ b/tests/distributed/raft/CMakeLists.txt @@ -15,10 +15,6 @@ foreach(test_cpp ${test_type_cpps}) # build exec file add_executable(${target_name} ${test_cpp}) - if(TEST_COVERAGE) - set_target_properties(${target_name} PROPERTIES COMPILE_FLAGS "-g -O0 -Wall -fprofile-arcs -ftest-coverage") - set_target_properties(${target_name} PROPERTIES LINK_FLAGS "--coverage -fprofile-arcs -ftest-coverage") - endif() # OUTPUT_NAME sets the real name of a target when it is built and can be # used to help create two targets of the same name even though CMake @@ -27,10 +23,6 @@ foreach(test_cpp ${test_type_cpps}) # link libraries target_link_libraries(${target_name} memgraph_lib) - if(TEST_COVERAGE) - # for code coverage - target_link_libraries(${target_name} gcov) - endif() set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index c77e6ce50..80b361844 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -5,6 +5,8 @@ get_filename_component(test_type ${CMAKE_CURRENT_SOURCE_DIR} NAME) file(GLOB_RECURSE test_type_cpps *.cpp) message(STATUS "Available ${test_type} cpp files are: ${test_type_cpps}") +add_custom_target(memgraph__unit) + # for each cpp file build binary and register test foreach(test_cpp ${test_type_cpps}) @@ -15,10 +17,6 @@ foreach(test_cpp ${test_type_cpps}) # build exec file add_executable(${target_name} ${test_cpp}) - if(TEST_COVERAGE) - set_target_properties(${target_name} PROPERTIES COMPILE_FLAGS "-g -O0 -Wall -fprofile-arcs -ftest-coverage") - set_target_properties(${target_name} PROPERTIES LINK_FLAGS "--coverage -fprofile-arcs -ftest-coverage") - endif() # OUTPUT_NAME sets the real name of a target when it is built and can be # used to help create two targets of the same name even though CMake @@ -29,13 +27,12 @@ foreach(test_cpp ${test_type_cpps}) target_link_libraries(${target_name} memgraph_lib) # gtest target_link_libraries(${target_name} gtest gmock gtest_main) - if(TEST_COVERAGE) - # for code coverage - target_link_libraries(${target_name} gcov) - endif() # register test set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml) add_test(${target_name} ${exec_name} --gtest_output=xml:${output_path}) + # add to memgraph__unit target + add_dependencies(memgraph__unit ${target_name}) + endforeach() diff --git a/tools/apollo/.gitignore b/tools/apollo/.gitignore index 160861f5e..3ef6de719 100644 --- a/tools/apollo/.gitignore +++ b/tools/apollo/.gitignore @@ -1 +1,2 @@ .cppcheck_errors +generated/* diff --git a/tools/apollo/apollo_archives.yaml b/tools/apollo/apollo_archives.yaml new file mode 100644 index 000000000..0af297daf --- /dev/null +++ b/tools/apollo/apollo_archives.yaml @@ -0,0 +1,7 @@ +- name: Code coverage + cd: generated/html + run_type: data process + archive: + - . + filename: code_coverage.tar.gz + host: true diff --git a/tools/apollo/apollo_runs.yaml b/tools/apollo/apollo_runs.yaml index 608a7da74..f4d4e4eb6 100644 --- a/tools/apollo/apollo_runs.yaml +++ b/tools/apollo/apollo_runs.yaml @@ -9,3 +9,16 @@ - ../../.clang-format # clang-format config file outfile_paths: - \./memgraph/tools/apollo/\.cppcheck_errors + +- name: code_coverage + project: ^mg-master-diff$ # regex to match only 'mg-master-diff' + type: data process + commands: ./coverage_convert + infiles: + - coverage_convert # coverage_convert script + - coverage_parse_export # coverage_parse_export script + - apollo_archives.yaml # coverage archive export config + - ../../src # src source dir + outfile_paths: + - \./memgraph/tools/apollo/generated/\.coverage_summary + - \./memgraph/tools/apollo/generated/\coverage.json diff --git a/tools/apollo/build_diff b/tools/apollo/build_diff index 8b1378917..e69de29bb 100644 --- a/tools/apollo/build_diff +++ b/tools/apollo/build_diff @@ -1 +0,0 @@ - diff --git a/tools/apollo/coverage_convert b/tools/apollo/coverage_convert new file mode 100755 index 000000000..6a168dd26 --- /dev/null +++ b/tools/apollo/coverage_convert @@ -0,0 +1,61 @@ +#!/bin/bash -e + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +project_dir="$( dirname "$( dirname "$script_dir" )" )" + +generated_dir="$script_dir/generated" +html_dir="$generated_dir/html" +data_file="$generated_dir/default.profdata" +json_file="$generated_dir/report.json" +coverage_file="$generated_dir/coverage.json" +summary_file="$generated_dir/.coverage_summary" + +# cleanup output directory +if [ -d "$generated_dir" ]; then + rm -rf "$generated_dir" +fi +mkdir "$generated_dir" + +# merge raw coverage info +raw_files="$( find "$HOME" -name "*.profraw" | tr '\n' ' ' )" +llvm-profdata-5.0 merge -sparse $raw_files -o "$data_file" + +# create list of binaries +cnt=0 +obj_files="" +for prof_file in $raw_files; do + if [ $cnt -gt 0 ]; then + obj_files+=" -object " + fi + obj_files+="$( find "$( dirname "$prof_file" )" -executable -type f | head -n 1 )" + cnt=$((cnt + 1)) +done + +# create list of source files +src_files=$( find "$HOME" \( -name '*.cpp' -o -name '*.hpp' \) -print | sort | tr '\n' ' ' ) + +# generate html output +llvm-cov-5.0 show $obj_files \ + -format html \ + -instr-profile "$data_file" \ + -o "$html_dir" \ + -show-line-counts-or-regions \ + -Xdemangler c++filt -Xdemangler -n \ + $src_files + +# fix names in html output +coverage_dir="$html_dir/coverage" +mv $coverage_dir/workspace/memgraph/* $html_dir/coverage/ +rm -r $coverage_dir/workspace +find $coverage_dir -name "*.html" -exec sed -i 's@/workspace/memgraph/@@g' {} \; +find $coverage_dir -name "*.html" -exec sed -i 's@../../style.css@style.css@g' {} \; +sed -i 's@/workspace/memgraph@@g' $html_dir/index.html + +# generate json output +llvm-cov-5.0 export $obj_files \ + -instr-profile "$data_file" \ + -Xdemangler c++filt -Xdemangler -n \ + $src_files > "$json_file" + +# process json output +$script_dir/coverage_parse_export "$json_file" "$coverage_file" "$summary_file" $src_files diff --git a/tools/apollo/coverage_parse_export b/tools/apollo/coverage_parse_export new file mode 100755 index 000000000..9fc9bfc49 --- /dev/null +++ b/tools/apollo/coverage_parse_export @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import sys +from collections import defaultdict + +def lines2phabricator(filename, lines): + ret = "" + numlines = 0 + with open(filename) as f: + for row in f: + numlines += 1 + for i in range(1, numlines + 1): + val = lines[i] + if val is None: ret += "N" + elif val == 0: ret += "U" + else: ret += "C" + return ret + +parser = argparse.ArgumentParser(description='Parse llvm-cov export data.') +parser.add_argument('input', help='input file') +parser.add_argument('coverage', help='coverage output file') +parser.add_argument('summary', help='summary output file') +parser.add_argument('files', nargs='+', help='files to process') +args = parser.parse_args() + +# for specification of the format see: +# https://github.com/llvm-mirror/llvm/blob/master/tools/llvm-cov/CoverageExporterJson.cpp + +data = json.load(open(args.input, "r")) + +totals = defaultdict(lambda: defaultdict(int)) +sources = defaultdict(lambda: defaultdict(lambda: None)) +for export in data["data"]: + for cfile in export["files"]: + for segment in cfile["segments"]: + filename = cfile["filename"] + if not filename in args.files: continue + line, col, count, has_count, is_region_entry = segment + sources[filename][line] = count + for function in export["functions"]: + for region in function["regions"]: + line_start, column_start, line_end, column_end, execution_count, \ + file_id, expanded_file_id, kind = region + filename = function["filenames"][file_id] + if filename not in args.files: continue + for i in range(line_start, line_end + 1): + sources[filename][i] = execution_count + for total, values in export["totals"].items(): + for key, value in values.items(): + totals[total][key] += value + +coverage = {} +for filename, lines in sources.items(): + path = "/".join(filename.split("/")[3:]) + coverage[path] = lines2phabricator(filename, lines) + +with open(args.coverage, "w") as f: + json.dump(coverage, f) + +summary = """==== Code coverage: ==== + + + +""" +ROW = "\n" +for what in ["functions", "instantiations", "lines", "regions"]: + now = totals[what] + now["percent"] = now["covered"] / now["count"] + summary += ROW.format(**dict(totals[what], name=what.capitalize())) +summary += "
CoverageTotal
{name}{covered} / {count} ({percent:.2%})
\n" + +with open(args.summary, "w") as f: + f.write(summary)