From 370f602921780eb109426d3f3f79e3f74d15d781 Mon Sep 17 00:00:00 2001 From: Matthias Volk Date: Thu, 15 Feb 2018 18:07:59 +0100 Subject: [PATCH] Travis: build Docker container with separate carl automatically --- .travis.yml | 71 +++++++++++++++++++++--------- travis/build-helper.sh | 21 +-------- travis/build.sh | 14 +++++- travis/build_carl.sh | 41 ++++++++++++++++++ travis/build_carl_helper.sh | 86 +++++++++++++++++++++++++++++++++++++ travis/generate_travis.py | 36 +++++++++++++++- 6 files changed, 226 insertions(+), 43 deletions(-) create mode 100755 travis/build_carl.sh create mode 100755 travis/build_carl_helper.sh diff --git a/.travis.yml b/.travis.yml index 559c002b8..16522980c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,37 @@ notifications: jobs: include: + ### + # Stage: Build Carl + ### + + # ubuntu-17.10 - DefaultDebugTravis + - stage: Build Carl + os: linux + compiler: gcc + env: CONFIG=DefaultDebugTravis LINUX=ubuntu-17.10 COMPILER=gcc + install: + - travis/install_linux.sh + script: + - travis/build_carl.sh + after_success: + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"; + - docker commit carl mvolk/carl-debug:travis; + - docker push mvolk/carl-debug:travis; + # ubuntu-17.10 - DefaultReleaseTravis + - stage: Build Carl + os: linux + compiler: gcc + env: CONFIG=DefaultReleaseTravis LINUX=ubuntu-17.10 COMPILER=gcc + install: + - travis/install_linux.sh + script: + - travis/build_carl.sh + after_success: + - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"; + - docker commit carl mvolk/carl:travis; + - docker push mvolk/carl:travis; + ### # Stage: Build (1st run) ### @@ -48,7 +79,7 @@ jobs: script: - travis/build.sh Build1 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # debian-9 - DefaultRelease @@ -62,7 +93,7 @@ jobs: script: - travis/build.sh Build1 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultDebugTravis @@ -76,7 +107,7 @@ jobs: script: - travis/build.sh Build1 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultReleaseTravis @@ -90,7 +121,7 @@ jobs: script: - travis/build.sh Build1 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; @@ -108,7 +139,7 @@ jobs: script: - travis/build.sh Build2 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # debian-9 - DefaultRelease @@ -121,7 +152,7 @@ jobs: script: - travis/build.sh Build2 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultDebugTravis @@ -134,7 +165,7 @@ jobs: script: - travis/build.sh Build2 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultReleaseTravis @@ -147,7 +178,7 @@ jobs: script: - travis/build.sh Build2 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; @@ -165,7 +196,7 @@ jobs: script: - travis/build.sh Build3 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # debian-9 - DefaultRelease @@ -178,7 +209,7 @@ jobs: script: - travis/build.sh Build3 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultDebugTravis @@ -191,7 +222,7 @@ jobs: script: - travis/build.sh Build3 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultReleaseTravis @@ -204,7 +235,7 @@ jobs: script: - travis/build.sh Build3 before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; @@ -222,7 +253,7 @@ jobs: script: - travis/build.sh BuildLast before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # debian-9 - DefaultRelease @@ -235,7 +266,7 @@ jobs: script: - travis/build.sh BuildLast before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultDebugTravis @@ -248,7 +279,7 @@ jobs: script: - travis/build.sh BuildLast before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultReleaseTravis @@ -261,7 +292,7 @@ jobs: script: - travis/build.sh BuildLast before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; @@ -279,7 +310,7 @@ jobs: script: - travis/build.sh TestAll before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # debian-9 - DefaultRelease @@ -292,7 +323,7 @@ jobs: script: - travis/build.sh TestAll before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; # ubuntu-17.10 - DefaultDebugTravis @@ -305,7 +336,7 @@ jobs: script: - travis/build.sh TestAll before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; after_success: @@ -322,7 +353,7 @@ jobs: script: - travis/build.sh TestAll before_cache: - - docker cp storm:/storm/. . + - docker cp storm:/opt/storm/. . after_failure: - find build -iname '*err*.log' -type f -print -exec cat {} \; after_success: diff --git a/travis/build-helper.sh b/travis/build-helper.sh index 2ba875bab..6da9ada64 100755 --- a/travis/build-helper.sh +++ b/travis/build-helper.sh @@ -23,23 +23,6 @@ run() { Build*) if [[ "$1" == "Build1" ]] then - - if [[ "$CONFIG" == "*Travis" ]] - then - # Build Carl separately - travis_fold start install_carl - cd .. - git clone https://github.com/smtrat/carl.git - cd carl - mkdir build - cd build - cmake .. "${CARL_CMAKE_ARGS[@]}" - make lib_carl -j$N_JOBS - cd ../../storm - travis_fold end install_carl - fi - - # CMake travis_fold start cmake mkdir build @@ -132,11 +115,9 @@ echo Normalized C++ Standard library location: $(readlink -f $(echo '#include /dev/null # Run container set -e - docker run -d -it --name storm --privileged mvolk/storm-basesystem:$LINUX + case "$CONFIG" in + *DebugTravis) + docker run -d -it --name storm --privileged mvolk/carl-debug:travis + ;; + *ReleaseTravis) + docker run -d -it --name storm --privileged mvolk/carl:travis + ;; + *) + docker run -d -it --name storm --privileged mvolk/storm-basesystem:$LINUX + ;; + esac # Copy local content into container docker exec storm mkdir opt/storm docker cp . storm:/opt/storm @@ -39,7 +49,7 @@ linux) export N_JOBS=$N_JOBS; export STLARG=; export OS=$OS; - cd opt/storm; + cd /opt/storm; travis/build-helper.sh $1" EXITCODE=$? ;; diff --git a/travis/build_carl.sh b/travis/build_carl.sh new file mode 100755 index 000000000..5054e3d6d --- /dev/null +++ b/travis/build_carl.sh @@ -0,0 +1,41 @@ +#!/bin/bash -x + +N_JOBS=2 +TIMEOUT_LINUX=2300 + +OS=$TRAVIS_OS_NAME + +case $OS in +linux) + # Execute docker image on Linux + # Stop previous session + docker rm -f carl &>/dev/null + # Run container + set -e + docker run -d -it --name carl --privileged mvolk/storm-basesystem:$LINUX + # Copy local content into container + docker cp travis/build_carl_helper.sh carl:/opt/ + set +e + + # Execute main process + timeout $TIMEOUT_LINUX docker exec carl bash -c " + export CONFIG=$CONFIG; + export COMPILER=$COMPILER; + export N_JOBS=$N_JOBS; + export STLARG=; + export OS=$OS; + cd /opt/; + ./build_carl_helper.sh" + ;; + +osx) + echo "Building carl on Mac OSX not used." + exit 1 + ;; + +*) + # Unknown OS + echo "Unsupported OS: $OS" + exit 1 +esac + diff --git a/travis/build_carl_helper.sh b/travis/build_carl_helper.sh new file mode 100755 index 000000000..cfb1a31a6 --- /dev/null +++ b/travis/build_carl_helper.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -e + +# Helper for travis folding +travis_fold() { + local action=$1 + local name=$2 + echo -en "travis_fold:${action}:${name}\r" +} + +# Helper for building and testing +run() { + travis_fold start install_dependencies + apt-get update + apt-get install -qq -y openjdk-8-jdk maven uuid-dev pkg-config + travis_fold end install_dependencies + + travis_fold start install_carl + git clone https://github.com/smtrat/carl.git + cd carl + mkdir build + cd build + cmake .. "${CMAKE_ARGS[@]}" + make lib_carl addons -j$N_JOBS + travis_fold end install_carl +} + + +# This only exists in OS X, but it doesn't cause issues in Linux (the dir doesn't exist, so it's +# ignored). +export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" + +case $COMPILER in +gcc-6) + export CC=gcc-6 + export CXX=g++-6 + ;; + +gcc) + export CC=gcc + export CXX=g++ + ;; + +clang-4) + case "$OS" in + linux) + export CC=clang-4.0 + export CXX=clang++-4.0 + ;; + osx) + export CC=/usr/local/opt/llvm/bin/clang-4.0 + export CXX=/usr/local/opt/llvm/bin/clang++ + ;; + *) echo "Error: unexpected OS: $OS"; exit 1 ;; + esac + ;; + +clang) + export CC=clang + export CXX=clang++ + ;; + +*) + echo "Unrecognized value of COMPILER: $COMPILER" + exit 1 +esac + +# Build +echo CXX version: $($CXX --version) +echo C++ Standard library location: $(echo '#include ' | $CXX -x c++ -E - | grep 'vector\"' | awk '{print $3}' | sed 's@/vector@@;s@\"@@g' | head -n 1) +echo Normalized C++ Standard library location: $(readlink -f $(echo '#include ' | $CXX -x c++ -E - | grep 'vector\"' | awk '{print $3}' | sed 's@/vector@@;s@\"@@g' | head -n 1)) + +case "$CONFIG" in +DefaultDebug*) + CMAKE_ARGS=(-DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="$STLARG" -DUSE_CLN_NUMBERS=ON -DUSE_GINAC=ON -DTHREAD_SAFE=ON -DBUILD_ADDONS=ON -DBUILD_ADDON_PARSER=ON) + ;; +DefaultRelease*) + CMAKE_ARGS=(-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="$STLARG" -DUSE_CLN_NUMBERS=ON -DUSE_GINAC=ON -DTHREAD_SAFE=ON -DBUILD_ADDONS=ON -DBUILD_ADDON_PARSER=ON) + ;; +*) + echo "Unrecognized value of CONFIG: $CONFIG"; exit 1 + ;; +esac + +run diff --git a/travis/generate_travis.py b/travis/generate_travis.py index 480da769d..4145711b4 100644 --- a/travis/generate_travis.py +++ b/travis/generate_travis.py @@ -65,6 +65,40 @@ if __name__ == "__main__": s += "jobs:\n" s += " include:\n" + # Start with prebuilding carl for docker + s += "\n" + s += " ###\n" + s += " # Stage: Build Carl\n" + s += " ###\n" + s += "\n" + for config in configs_linux: + linux = config[0] + compiler = config[1] + build_type = config[2] + if "Travis" in build_type: + s += " # {} - {}\n".format(linux, build_type) + buildConfig = "" + buildConfig += " - stage: Build Carl\n" + buildConfig += " os: linux\n" + buildConfig += " compiler: {}\n".format(compiler) + buildConfig += " env: CONFIG={} LINUX={} COMPILER={}\n".format(build_type, linux, compiler) + buildConfig += " install:\n" + buildConfig += " - travis/install_linux.sh\n" + buildConfig += " script:\n" + buildConfig += " - travis/build_carl.sh\n" + # Upload to DockerHub + buildConfig += " after_success:\n" + buildConfig += ' - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD";\n' + if "Debug" in build_type: + buildConfig += " - docker commit carl mvolk/carl-debug:travis;\n" + buildConfig += " - docker push mvolk/carl-debug:travis;\n" + elif "Release" in build_type: + buildConfig += " - docker commit carl mvolk/carl:travis;\n" + buildConfig += " - docker push mvolk/carl:travis;\n" + else: + assert False + s += buildConfig + # Generate all configurations for stage in stages: s += "\n" @@ -112,7 +146,7 @@ if __name__ == "__main__": buildConfig += " script:\n" buildConfig += " - travis/build.sh {}\n".format(stage[1]) buildConfig += " before_cache:\n" - buildConfig += " - docker cp storm:/storm/. .\n" + buildConfig += " - docker cp storm:/opt/storm/. .\n" buildConfig += " after_failure:\n" buildConfig += " - find build -iname '*err*.log' -type f -print -exec cat {} \;\n" # Upload to DockerHub