From 2fdbabc13c2b06c949c2212de17f3d43389c53ad Mon Sep 17 00:00:00 2001 From: whitequark Date: Mon, 25 Jul 2016 22:09:58 +0000 Subject: [PATCH] Add statement/condition coverage measurement for the test suite. We're using gcov+lcov, since these tools appear to be the only usable ones that use the SC/CC metric; and measuring just the line coverage would be practically criminal negligence. gcov only works with GCC and Clang, and MSVC's own coverage measurement tools are not up to the task; so MSVC is out of luck. --- .travis/build-debian.sh | 3 ++- .travis/install-debian.sh | 2 +- CMakeLists.txt | 35 +++++++++++++++++++++++++++++++---- src/CMakeLists.txt | 9 ++++++++- src/solvespace.h | 1 + test/CMakeLists.txt | 38 +++++++++++++++++++++++++++++++++++++- 6 files changed, 80 insertions(+), 8 deletions(-) diff --git a/.travis/build-debian.sh b/.travis/build-debian.sh index f15b8846..2b6db43c 100755 --- a/.travis/build-debian.sh +++ b/.travis/build-debian.sh @@ -4,5 +4,6 @@ if echo $TRAVIS_TAG | grep ^v; then BUILD_TYPE=RelWithDebInfo; else BUILD_TYPE=D mkdir build cd build -cmake -DCMAKE_C_COMPILER=gcc-5 -DCMAKE_CXX_COMPILER=g++-5 -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. +cmake -DCMAKE_C_COMPILER=gcc-5 -DCMAKE_CXX_COMPILER=g++-5 \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DENABLE_COVERAGE=ON .. make VERBOSE=1 diff --git a/.travis/install-debian.sh b/.travis/install-debian.sh index 1804276e..90080f53 100755 --- a/.travis/install-debian.sh +++ b/.travis/install-debian.sh @@ -5,4 +5,4 @@ sudo apt-get update -qq sudo apt-get install -q -y \ cmake cmake-data libpng12-dev zlib1g-dev libjson0-dev libfontconfig1-dev \ libgtkmm-2.4-dev libpangomm-1.4-dev libcairo2-dev libgl1-mesa-dev libglu-dev \ - libfreetype6-dev dpkg-dev gcc-5 g++-5 + libfreetype6-dev dpkg-dev gcc-5 g++-5 lcov diff --git a/CMakeLists.txt b/CMakeLists.txt index 84a29b7a..15f23659 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,10 @@ set(solvespace_VERSION_MAJOR 3) set(solvespace_VERSION_MINOR 0) string(SUBSTRING "${GIT_COMMIT_HASH}" 0 8 solvespace_GIT_HASH) -set(ENABLE_TESTS ON CACHE BOOL "Whether the test suite will be built and run") +set(ENABLE_TESTS ON CACHE BOOL + "Whether the test suite will be built and run") +set(ENABLE_COVERAGE OFF CACHE BOOL + "Whether code coverage information will be collected") if(NOT WIN32 AND NOT APPLE) set(GUI gtk2 CACHE STRING "GUI toolkit to use (one of: gtk2 gtk3)") @@ -56,7 +59,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") endif() if(SANITIZE) - if(NOT (CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")) + if(NOT (CMAKE_C_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) message(ERROR "Sanitizers are only available in Clang/Clang++") endif() set(SANITIZE_FLAGS "-O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls") @@ -162,6 +165,14 @@ else() # Linux and compatible systems endif() endif() +if(ENABLE_COVERAGE) + find_program(LCOV lcov) + find_program(GENHTML genhtml) + if(NOT LCOV OR NOT GENHTML) + message(FATAL_ERROR "lcov is required for producing coverage reports") + endif() +endif() + # solvespace-only compiler flags if(WIN32) @@ -193,9 +204,9 @@ if(MSVC) set(WARNING_FLAGS "${WARNING_FLAGS} /we4062") endif() -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(WARNING_FLAGS "-Wall -Wextra -Wno-unused-parameter") - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(WARNING_FLAGS "${WARNING_FLAGS} -Wfloat-conversion") endif() # We rely on these -Werror flags. @@ -209,6 +220,22 @@ if(WIN32) set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -l0") endif() +if(ENABLE_COVERAGE) + if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR + CMAKE_CXX_COMPILER_ID MATCHES "Clang")) + message(FATAL_ERROR "Code coverage is only available on GCC and Clang") + endif() + + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(FATAL_ERROR "Code coverage produces reliable results only on Debug builds") + endif() + + # With -fexceptions, every call becomes a branch. While technically accurate, + # this is not useful for us. + set(COVERAGE_FLAGS -fno-exceptions --coverage) + set(COVERAGE_LIBRARY --coverage) +endif() + # components add_subdirectory(res) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ab49c007..482c3085 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -184,6 +184,9 @@ target_link_libraries(solvespace_cad ${FREETYPE_LIBRARY} ${Backtrace_LIBRARIES}) +target_compile_options(solvespace_cad + PRIVATE ${COVERAGE_FLAGS}) + # solvespace gui executable add_executable(solvespace WIN32 MACOSX_BUNDLE @@ -197,7 +200,8 @@ add_dependencies(solvespace target_link_libraries(solvespace solvespace_cad ${OPENGL_LIBRARIES} - ${platform_LIBRARIES}) + ${platform_LIBRARIES} + ${COVERAGE_LIBRARY}) if(WIN32 AND NOT MINGW) set_target_properties(solvespace PROPERTIES @@ -256,3 +260,6 @@ target_include_directories(solvespace_headless target_link_libraries(solvespace_headless solvespace_cad ${CAIRO_LIBRARIES}) + +target_compile_options(solvespace_headless + PRIVATE ${COVERAGE_FLAGS}) diff --git a/src/solvespace.h b/src/solvespace.h index e5bf07d6..03c8a6e9 100644 --- a/src/solvespace.h +++ b/src/solvespace.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8b8492dd..ce94b1d2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,3 +1,5 @@ +# test suite + include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) @@ -10,9 +12,43 @@ add_executable(solvespace_testsuite ${testsuite_SOURCES}) target_link_libraries(solvespace_testsuite - solvespace_headless) + solvespace_headless + ${COVERAGE_LIBRARY}) add_custom_target(solvespace-test ALL COMMAND $ COMMENT "Testing SolveSpace" VERBATIM) + +# coverage reports + +if(ENABLE_COVERAGE) + set(LCOV_FLAGS -q --rc lcov_branch_coverage=1 --rc lcov_excl_line=ssassert) + set(LCOV_COLLECT -c -b ${CMAKE_SOURCE_DIR}/src -d ${CMAKE_BINARY_DIR}/src --no-external) + + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/coverage_base.info + COMMAND ${LCOV} ${LCOV_FLAGS} ${LCOV_COLLECT} + -o ${CMAKE_BINARY_DIR}/coverage_base.info -i + DEPENDS solvespace_testsuite + COMMENT "Importing baseline coverage data" + VERBATIM) + + add_custom_target(solvespace-coverage ALL + COMMAND ${LCOV} ${LCOV_FLAGS} ${LCOV_COLLECT} + -o ${CMAKE_BINARY_DIR}/coverage_test.info + COMMAND ${LCOV} ${LCOV_FLAGS} + -o ${CMAKE_BINARY_DIR}/coverage_full.info + -a ${CMAKE_BINARY_DIR}/coverage_base.info + -a ${CMAKE_BINARY_DIR}/coverage_test.info + COMMAND ${LCOV} ${LCOV_FLAGS} --summary + ${CMAKE_BINARY_DIR}/coverage_full.info + COMMAND ${GENHTML} -q --branch-coverage --demangle-cpp --legend + ${CMAKE_BINARY_DIR}/coverage_full.info + -o ${CMAKE_BINARY_DIR}/coverage/ + -t "SolveSpace testbench" + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/coverage_base.info + DEPENDS solvespace-test + COMMENT "Generating coverage report" + VERBATIM) +endif()