Add docker configurations used by the buildbots.

These are the scripts I use to create the docker images for
the build bots and run them.

git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@347244 91177308-0d34-0410-b5e6-96231b3b80d8

utils/docker/build_docker_image.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+#===- libcxx/utils/docker/build_docker_image.sh ----------------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===----------------------------------------------------------------------===//
+set -e
+
+IMAGE_SOURCE=""
+DOCKER_REPOSITORY=""
+DOCKER_TAG=""
+
+function show_usage() {
+  cat << EOF
+Usage: build_docker_image.sh [options] [-- [cmake_args]...]
+
+Available options:
+  General:
+    -h|--help               show this help message
+  Docker-specific:
+    -s|--source             image source dir (i.e. debian8, nvidia-cuda, etc)
+    -d|--docker-repository  docker repository for the image
+    -t|--docker-tag         docker tag for the image
+
+Required options: --source and --docker-repository.
+
+For example, running:
+$ build_docker_image.sh -s debian9 -d mydocker/debian9-clang -t latest
+will produce two docker images:
+    mydocker/debian9-clang-build:latest - an intermediate image used to compile
+      clang.
+    mydocker/clang-debian9:latest       - a small image with preinstalled clang.
+Please note that this example produces a not very useful installation, since it
+doesn't override CMake defaults, which produces a Debug and non-boostrapped
+version of clang.
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    -s|--source)
+      shift
+      IMAGE_SOURCE="$1"
+      shift
+      ;;
+    -d|--docker-repository)
+      shift
+      DOCKER_REPOSITORY="$1"
+      shift
+      ;;
+    -t|--docker-tag)
+      shift
+      DOCKER_TAG="$1"
+      shift
+      ;;
+    *)
+      echo "Unknown argument $1"
+      exit 1
+      ;;
+  esac
+done
+
+
+command -v docker >/dev/null ||
+  {
+    echo "Docker binary cannot be found. Please install Docker to use this script."
+    exit 1
+  }
+
+if [ "$IMAGE_SOURCE" == "" ]; then
+  echo "Required argument missing: --source"
+  exit 1
+fi
+
+if [ "$DOCKER_REPOSITORY" == "" ]; then
+  echo "Required argument missing: --docker-repository"
+  exit 1
+fi
+
+SOURCE_DIR=$(dirname $0)
+if [ ! -d "$SOURCE_DIR/$IMAGE_SOURCE" ]; then
+  echo "No sources for '$IMAGE_SOURCE' were found in $SOURCE_DIR"
+  exit 1
+fi
+
+BUILD_DIR=$(mktemp -d)
+trap "rm -rf $BUILD_DIR" EXIT
+echo "Using a temporary directory for the build: $BUILD_DIR"
+
+cp -r "$SOURCE_DIR/$IMAGE_SOURCE" "$BUILD_DIR/$IMAGE_SOURCE"
+cp -r "$SOURCE_DIR/scripts" "$BUILD_DIR/scripts"
+
+
+if [ "$DOCKER_TAG" != "" ]; then
+  DOCKER_TAG=":$DOCKER_TAG"
+fi
+
+echo "Building ${DOCKER_REPOSITORY}${DOCKER_TAG} from $IMAGE_SOURCE"
+docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \
+  -f "$BUILD_DIR/$IMAGE_SOURCE/Dockerfile" \
+  "$BUILD_DIR"
+echo "Done"

utils/docker/debian9/Dockerfile

@@ -0,0 +1,113 @@
+#===- libcxx/utils/docker/debian9/Dockerfile -------------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===----------------------------------------------------------------------===//
+
+# Setup the base builder image with the packages we'll need to build GCC and Clang from source.
+FROM launcher.gcr.io/google/debian9:latest as builder-base
+LABEL maintainer "libc++ Developers"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      ca-certificates \
+      gnupg \
+      build-essential \
+      wget \
+      subversion \
+      unzip \
+      automake \
+      python \
+      cmake \
+      ninja-build \
+      curl \
+      git \
+      gcc-multilib \
+      g++-multilib \
+      libc6-dev \
+      bison \
+      flex \
+      libtool \
+      autoconf \
+      binutils-dev \
+      binutils-gold \
+      software-properties-common && \
+  update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.gold" 20 && \
+  update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.bfd" 10
+
+# Build GCC 4.9 for testing our C++11 against
+FROM builder-base as gcc-49-builder
+LABEL maintainer "libc++ Developers"
+
+ADD scripts/build_gcc.sh /tmp/build_gcc.sh
+
+RUN git clone --depth=1 --branch gcc-4_9_4-release git://gcc.gnu.org/git/gcc.git /tmp/gcc-4.9.4
+RUN cd /tmp/gcc-4.9.4/ && ./contrib/download_prerequisites
+RUN /tmp/build_gcc.sh --source /tmp/gcc-4.9.4 --to /opt/gcc-4.9.4
+
+# Build GCC ToT for testing in all dialects.
+FROM builder-base as gcc-tot-builder
+LABEL maintainer "libc++ Developers"
+
+ADD scripts/build_gcc.sh /tmp/build_gcc.sh
+
+RUN git clone --depth=1 git://gcc.gnu.org/git/gcc.git /tmp/gcc-tot
+RUN cd /tmp/gcc-tot && ./contrib/download_prerequisites
+RUN /tmp/build_gcc.sh --source /tmp/gcc-tot --to /opt/gcc-tot
+
+# Build LLVM 4.0 which is used to test against a "legacy" compiler.
+FROM builder-base as llvm-4-builder
+LABEL maintainer "libc++ Developers"
+
+ADD scripts/checkout_git.sh /tmp/checkout_git.sh
+ADD scripts/build_install_llvm.sh /tmp/build_install_llvm.sh
+
+RUN /tmp/checkout_git.sh --to /tmp/llvm-4.0 -p clang -p compiler-rt --branch release_40
+RUN /tmp/build_install_llvm.sh \
+    --install /opt/llvm-4.0 \
+    --source /tmp/llvm-4.0 \
+    --build /tmp/build-llvm-4.0 \
+    -i install-clang -i install-clang-headers \
+    -i install-compiler-rt \
+    -- \
+    -DCMAKE_BUILD_TYPE=RELEASE \
+    -DLLVM_ENABLE_ASSERTIONS=ON
+
+# Stage 2. Produce a minimal release image with build results.
+FROM launcher.gcr.io/google/debian9:latest
+LABEL maintainer "libc++ Developers"
+
+# Copy over the GCC and Clang installations
+COPY --from=gcc-49-builder /opt/gcc-4.9.4 /opt/gcc-4.9.4
+COPY --from=gcc-tot-builder /opt/gcc-tot /opt/gcc-tot
+COPY --from=llvm-4-builder /opt/llvm-4.0 /opt/llvm-4.0
+
+RUN ln -s /opt/gcc-4.9.4/bin/gcc /usr/local/bin/gcc-4.9 && \
+    ln -s /opt/gcc-4.9.4/bin/g++ /usr/local/bin/g++-4.9
+
+RUN apt-get update && \
+    apt-get install -y \
+      ca-certificates \
+      gnupg \
+      build-essential \
+      apt-transport-https \
+      curl \
+      software-properties-common
+
+RUN apt-get install -y --no-install-recommends \
+    systemd \
+    sysvinit-utils \
+    cmake \
+    subversion \
+    git \
+    ninja-build \
+    gcc-multilib \
+    g++-multilib \
+    python \
+    buildbot-slave
+
+ADD scripts /libcxx-scripts/
+RUN /libcxx-scripts/install_clang_packages.sh

utils/docker/scripts/build_gcc.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+#===- libcxx/utils/docker/scripts/build-gcc.sh ----------------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===-----------------------------------------------------------------------===//
+
+set -e
+
+
+function show_usage() {
+  cat << EOF
+Usage: build-gcc.sh [options]
+
+Run autoconf with the specified arguments. Used inside docker container.
+
+Available options:
+  -h|--help           show this help message
+  --source            the source path from which to run the configuration.
+  --to                destination directory where to install the targets.
+Required options: --to, at least one --install-target.
+
+All options after '--' are passed to CMake invocation.
+EOF
+}
+
+GCC_INSTALL_DIR=""
+GCC_SOURCE_DIR=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --to)
+      shift
+      GCC_INSTALL_DIR="$1"
+      shift
+      ;;
+    --source)
+      shift
+      GCC_SOURCE_DIR="$1"
+      shift
+      ;;
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+  esac
+done
+
+if [ "$GCC_INSTALL_DIR" == "" ]; then
+  echo "No install directory. Please specify the --to argument."
+  exit 1
+fi
+
+if [ "$GCC_SOURCE_DIR" == "" ]; then
+  echo "No source directory. Please specify the --source argument."
+  exit 1
+fi
+
+GCC_NAME=`basename $GCC_SOURCE_DIR`
+GCC_BUILD_DIR="/tmp/gcc-build-root/build-$GCC_NAME"
+
+mkdir -p "$GCC_INSTALL_DIR"
+mkdir -p "$GCC_BUILD_DIR"
+pushd "$GCC_BUILD_DIR"
+
+# Run the build as specified in the build arguments.
+echo "Running configuration"
+$GCC_SOURCE_DIR/configure --prefix=$GCC_INSTALL_DIR \
+  --disable-bootstrap --disable-libgomp --disable-libitm \
+  --disable-libvtv --disable-libcilkrts --disable-libmpx \
+  --disable-liboffloadmic --disable-libcc1 --enable-languages=c,c++
+
+NPROC=`nproc`
+echo "Running build with $NPROC threads"
+make -j$NPROC
+
+echo "Installing to $GCC_INSTALL_DIR"
+make install -j$NPROC
+
+popd
+
+# Cleanup.
+rm -rf "$GCC_BUILD_DIR"
+
+echo "Done"

utils/docker/scripts/build_install_llvm.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+#===- llvm/utils/docker/scripts/build_install_llvm.sh ---------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===-----------------------------------------------------------------------===//
+
+set -e
+
+function show_usage() {
+  cat << EOF
+Usage: build_install_llvm.sh [options] -- [cmake-args]
+
+Run cmake with the specified arguments. Used inside docker container.
+Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into
+the directory specified by --to option.
+
+Available options:
+  -h|--help           show this help message
+  -i|--install-target name of a cmake install target to build and include in
+                      the resulting archive. Can be specified multiple times.
+  --install           destination directory where to install the targets.
+  --source            location of the source tree.
+  --build             location to use as the build directory.
+Required options: --to, --source, --build, and at least one --install-target.
+
+All options after '--' are passed to CMake invocation.
+EOF
+}
+
+CMAKE_ARGS=""
+CMAKE_INSTALL_TARGETS=""
+CLANG_INSTALL_DIR=""
+CLANG_SOURCE_DIR=""
+CLANG_BUILD_DIR=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -i|--install-target)
+      shift
+      CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1"
+      shift
+      ;;
+    --source)
+      shift
+      CLANG_SOURCE_DIR="$1"
+      shift
+      ;;
+    --build)
+      shift
+      CLANG_BUILD_DIR="$1"
+      shift
+      ;;
+    --install)
+      shift
+      CLANG_INSTALL_DIR="$1"
+      shift
+      ;;
+    --)
+      shift
+      CMAKE_ARGS="$*"
+      shift $#
+      ;;
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+  esac
+done
+
+if [ "$CLANG_SOURCE_DIR" == "" ]; then
+  echo "No source directory. Please pass --source."
+  exit 1
+fi
+
+if [ "$CLANG_BUILD_DIR" == "" ]; then
+  echo "No build directory. Please pass --build"
+  exit 1
+fi
+
+if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then
+  echo "No install targets. Please pass one or more --install-target."
+  exit 1
+fi
+
+if [ "$CLANG_INSTALL_DIR" == "" ]; then
+  echo "No install directory. Please specify the --to argument."
+  exit 1
+fi
+
+echo "Building in $CLANG_BUILD_DIR"
+mkdir -p "$CLANG_BUILD_DIR"
+pushd "$CLANG_BUILD_DIR"
+
+# Run the build as specified in the build arguments.
+echo "Running build"
+cmake -GNinja \
+  -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \
+  $CMAKE_ARGS \
+  "$CLANG_SOURCE_DIR"
+ninja $CMAKE_INSTALL_TARGETS
+
+popd
+
+# Cleanup.
+rm -rf "$CLANG_BUILD_DIR"
+
+echo "Done"

utils/docker/scripts/checkout_git.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+#===- llvm/utils/docker/scripts/checkout.sh ---------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===-----------------------------------------------------------------------===//
+
+set -e
+
+function show_usage() {
+  cat << EOF
+Usage: checkout.sh [options]
+
+Checkout svn sources into /tmp/clang-build/src. Used inside a docker container.
+
+Available options:
+  -h|--help           show this help message
+  -b|--branch         svn branch to checkout, i.e. 'trunk',
+                      'branches/release_40'
+                      (default: 'trunk')
+  -p|--llvm-project   name of an svn project to checkout.
+                      For clang, please use 'clang', not 'cfe'.
+                      Project 'llvm' is always included and ignored, if
+                      specified.
+                      Can be specified multiple times.
+EOF
+}
+
+LLVM_BRANCH=""
+# We always checkout llvm
+LLVM_PROJECTS="llvm"
+SOURCE_DIR=""
+
+function contains_project() {
+  local TARGET_PROJ="$1"
+  local PROJ
+  for PROJ in $LLVM_PROJECTS; do
+    if [ "$PROJ" == "$TARGET_PROJ" ]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --to)
+      shift
+      SOURCE_DIR="$1"
+      shift
+      ;;
+    -b|--branch)
+      shift
+      LLVM_BRANCH="$1"
+      shift
+      ;;
+    -p|--llvm-project)
+      shift
+      PROJ="$1"
+      shift
+
+      if [ "$PROJ" == "cfe" ]; then
+        PROJ="clang"
+      fi
+
+      if ! contains_project "$PROJ" ; then
+        if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then
+          echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first."
+          LLVM_PROJECTS="$LLVM_PROJECTS clang"
+        fi
+        LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
+      else
+        echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
+      fi
+      ;;
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+  esac
+done
+
+if [ "$SOURCE_DIR" == "" ]; then
+  echo "Must specify checkout directory using --to"
+  exit 1
+fi
+
+if [ "$LLVM_BRANCH" == "" ]; then
+  GIT_BRANCH_ARG=""
+else
+  GIT_BRANCH_ARG="--branch $LLVM_BRANCH"
+fi
+
+if [ "$LLVM_SVN_REV" != "" ]; then
+  SVN_REV_ARG="-r$LLVM_SVN_REV"
+  echo "Checking out svn revision r$LLVM_SVN_REV."
+else
+  SVN_REV_ARG=""
+  echo "Checking out latest svn revision."
+fi
+
+# Get the sources from svn.
+echo "Checking out sources from git"
+
+for LLVM_PROJECT in $LLVM_PROJECTS; do
+  if [ "$LLVM_PROJECT" == "llvm" ]; then
+    CHECKOUT_DIR="$SOURCE_DIR"
+  elif [ "$LLVM_PROJECT" == "libcxx" ] || [ "$LLVM_PROJECT" == "libcxxabi" ] || [ "$LLVM_PROJECT" == "compiler-rt" ]; then
+    CHECKOUT_DIR="$SOURCE_DIR/projects/$LLVM_PROJECT"
+  elif [ "$LLVM_PROJECT" == "clang" ]; then
+    CHECKOUT_DIR="$SOURCE_DIR/tools/clang"
+  elif [ "$LLVM_PROJECT" == "clang-tools-extra" ]; then
+    CHECKOUT_DIR="$SOURCE_DIR/tools/clang/tools/extra"
+  else
+    CHECKOUT_DIR="$SOURCE_DIR/$LLVM_PROJECT"
+  fi
+
+  echo "Checking out https://git.llvm.org/git/$LLVM_PROJECT to $CHECKOUT_DIR"
+  git clone --depth=1 $GIT_BRANCH_ARG \
+    "https://git.llvm.org/git/$LLVM_PROJECT.git" \
+    "$CHECKOUT_DIR"
+done
+
+echo "Done"

utils/docker/scripts/install_clang_packages.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#===- libcxx/utils/docker/scripts/install_clang_package.sh -----------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===-----------------------------------------------------------------------===//
+
+set -e
+
+function show_usage() {
+  cat << EOF
+Usage: install_clang_package.sh [options]
+
+Install
+Available options:
+  -h|--help           show this help message
+  --version           the numeric version of the package to use.
+EOF
+}
+
+VERSION=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --version)
+      shift
+      VERSION="$1"
+      shift
+      ;;
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+  esac
+done
+
+
+
+curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+add-apt-repository -s "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main"
+apt-get update
+apt-get install -y --no-install-recommends clang
+
+echo "Testing clang version..."
+clang --version
+
+echo "Testing clang++ version..."
+clang++ --version
+
+# Figure out the libc++ and libc++abi package versions that we want.
+if [ "$VERSION" == "" ]; then
+  VERSION="$(apt-cache search 'libc\+\+-[0-9]-dev' | awk '{print $1}' | awk -F- '{print $2}')"
+  echo "Installing version '$VERSION'"
+fi
+
+apt-get install -y --no-install-recommends "libc++-$VERSION-dev" "libc++abi-$VERSION-dev"
+
+echo "Done"

utils/docker/scripts/run_buildbot.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -x
+
+BOT_DIR=/b
+BOT_NAME=$1
+BOT_PASS=$2
+
+mkdir -p $BOT_DIR
+
+#curl "https://repo.stackdriver.com/stack-install.sh" | bash -s -- --write-gcm
+
+apt-get update -y
+apt-get upgrade -y
+
+systemctl set-property buildslave.service TasksMax=100000
+
+buildslave stop $BOT_DIR
+
+chown buildbot:buildbot $BOT_DIR
+
+echo "Connecting as $BOT_NAME"
+buildslave create-slave --allow-shutdown=signal $BOT_DIR lab.llvm.org:9990 $BOT_NAME $BOT_PASS
+
+echo "Eric Fiselier <ericwf@google.com>" > $BOT_DIR/info/admin
+
+{
+  uname -a | head -n1
+  cmake --version | head -n1
+  g++ --version | head -n1
+  ld --version | head -n1
+  date
+  lscpu
+} > $BOT_DIR/info/host
+
+echo "SLAVE_RUNNER=/usr/bin/buildslave
+SLAVE_ENABLED[1]=\"1\"
+SLAVE_NAME[1]=\"buildslave1\"
+SLAVE_USER[1]=\"buildbot\"
+SLAVE_BASEDIR[1]=\"$BOT_DIR\"
+SLAVE_OPTIONS[1]=\"\"
+SLAVE_PREFIXCMD[1]=\"\"" > /etc/default/buildslave
+
+chown -R buildbot:buildbot $BOT_DIR
+systemctl daemon-reload
+service buildslave restart
+
+sleep 30
+cat $BOT_DIR/twistd.log
+grep "slave is ready" $BOT_DIR/twistd.log || shutdown now
+
+# GCE can restart instance after 24h in the middle of the build.
+# Gracefully restart before that happen.
+sleep 72000
+while pkill -SIGHUP buildslave; do sleep 5; done;
+shutdown now