[Dockerfiles] Split checkout and build scripts into separate files.

Summary:
This is a small refactoring to extract the svn checkout code from the
build script used inside the docker image.
This would give more flexibility if more than a single invocation of
cmake is needed inside the docker image.

User-facing interface (build_docker_image.sh) hasn't changed, only the
internal scripts running inside the build container are affected.

Reviewers: ioeric

Reviewed By: ioeric

Subscribers: mehdi_amini, llvm-commits

Differential Revision: https://reviews.llvm.org/D45868

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@330412 91177308-0d34-0410-b5e6-96231b3b80d8

utils/docker/build_docker_image.sh

@@ -13,6 +13,8 @@ IMAGE_SOURCE=""
 DOCKER_REPOSITORY=""
 DOCKER_TAG=""
 BUILDSCRIPT_ARGS=""
+CHECKOUT_ARGS=""
+CMAKE_ENABLED_PROJECTS=""
 
 function show_usage() {
   cat << EOF
@@ -25,7 +27,7 @@ Available options:
     -s|--source             image source dir (i.e. debian8, nvidia-cuda, etc)
     -d|--docker-repository  docker repository for the image
     -t|--docker-tag         docker tag for the image
-  LLVM-specific:
+  Checkout arguments:
     -b|--branch         svn branch to checkout, i.e. 'trunk',
                         'branches/release_40'
                         (default: 'trunk')
@@ -40,11 +42,12 @@ Available options:
                         Project 'llvm' is always included and ignored, if
                         specified.
                         Can be specified multiple times.
-    -i|--install-target name of a cmake install target to build and include in
-                        the resulting archive. Can be specified multiple times.
     -c|--checksums      name of a file, containing checksums of llvm checkout.
                         Script will fail if checksums of the checkout do not
                         match.
+  Build-specific:
+    -i|--install-target name of a cmake install target to build and include in
+                        the resulting archive. Can be specified multiple times.
 
 Required options: --source and --docker-repository, at least one
   --install-target.
@@ -75,6 +78,7 @@ EOF
 
 CHECKSUMS_FILE=""
 SEEN_INSTALL_TARGET=0
+SEEN_CMAKE_ARGS=0
 while [[ $# -gt 0 ]]; do
   case "$1" in
     -h|--help)
@@ -96,11 +100,26 @@ while [[ $# -gt 0 ]]; do
       DOCKER_TAG="$1"
       shift
       ;;
-    -i|--install-target|-r|--revision|-c|-cherrypick|-b|--branch|-p|--llvm-project)
-      if [ "$1" == "-i" ] || [ "$1" == "--install-target" ]; then
-        SEEN_INSTALL_TARGET=1
-      fi
+    -r|--revision|-c|-cherrypick|-b|--branch)
+      CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $2"
+      shift 2
+      ;;
+    -i|--install-target)
+      SEEN_INSTALL_TARGET=1
       BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2"
+      shift 2
+      ;;
+    -p|--llvm-project)
+      PROJ="$2"
+      if [ "$PROJ" == "cfe" ]; then
+        PROJ="clang"
+      fi
+
+      CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $PROJ"
+      if [ "$PROJ" != "clang-tools-extra" ]; then
+        CMAKE_ENABLED_PROJECTS="$CMAKE_ENABLED_PROJECTS:$PROJ"
+      fi
+
       shift 2
       ;;
     -c|--checksums)
@@ -111,6 +130,7 @@ while [[ $# -gt 0 ]]; do
     --)
       shift
       BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*"
+      SEEN_CMAKE_ARGS=1
       shift $#
       ;;
     *)
@@ -120,6 +140,17 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
+
+if [ "$CMAKE_ENABLED_PROJECTS" != "" ]; then
+  # Remove the leading ':' character.
+  CMAKE_ENABLED_PROJECTS="${CMAKE_ENABLED_PROJECTS:1}"
+
+  if [[ $SEEN_CMAKE_ARGS -eq 0 ]]; then
+    BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS --"
+  fi
+  BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -DLLVM_ENABLE_PROJECTS=$CMAKE_ENABLED_PROJECTS"
+fi
+
 command -v docker >/dev/null ||
   {
     echo "Docker binary cannot be found. Please install Docker to use this script."
@@ -165,6 +196,7 @@ fi
 
 echo "Building ${DOCKER_REPOSITORY}${DOCKER_TAG} from $IMAGE_SOURCE"
 docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \
+  --build-arg "checkout_args=$CHECKOUT_ARGS" \
   --build-arg "buildscript_args=$BUILDSCRIPT_ARGS" \
   -f "$BUILD_DIR/$IMAGE_SOURCE/Dockerfile" \
   "$BUILD_DIR"

utils/docker/debian8/Dockerfile

@@ -41,10 +41,13 @@ RUN mkdir /tmp/cmake-install && cd /tmp/cmake-install && \
 
 ADD checksums /tmp/checksums
 ADD scripts /tmp/scripts
-# Arguments passed to build_install_clang.sh.
-ARG buildscript_args
+
+# Checkout the source code.
+ARG checkout_args
+RUN /tmp/scripts/checkout.sh ${checkout_args}
 # Run the build. Results of the build will be available at /tmp/clang-install/.
-RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
+ARG buildscript_args
+RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
 
 
 # Stage 2. Produce a minimal release image with build results.

utils/docker/example/Dockerfile

@@ -19,10 +19,13 @@ LABEL maintainer "Maintainer <maintainer@email>"
 
 ADD checksums /tmp/checksums
 ADD scripts /tmp/scripts
-# Arguments passed to build_install_clang.sh.
+
+# Checkout the source code.
+ARG checkout_args
+RUN /tmp/scripts/checkout.sh ${checkout_args}
+# Run the build. Results of the build will be available at /tmp/clang-install/.
 ARG buildscript_args
-# Run the build. Results of the build will be available as /tmp/clang-install.
-RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
 
 
 # Stage 2. Produce a minimal release image with build results.

utils/docker/nvidia-cuda/Dockerfile

@@ -17,10 +17,13 @@ RUN apt-get update && \
 
 ADD checksums /tmp/checksums
 ADD scripts /tmp/scripts
-# Arguments passed to build_install_clang.sh.
-ARG buildscript_args
+
+# Checkout the source code.
+ARG checkout_args
+RUN /tmp/scripts/checkout.sh ${checkout_args}
 # Run the build. Results of the build will be available at /tmp/clang-install/.
-RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
+ARG buildscript_args
+RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
 
 
 # Stage 2. Produce a minimal release image with build results.

utils/docker/scripts/build_install_llvm.sh

@@ -14,112 +14,35 @@ function show_usage() {
   cat << EOF
 Usage: build_install_llvm.sh [options] -- [cmake-args]
 
-Checkout svn sources and run cmake with the specified arguments. Used
-inside docker container.
+Run cmake with the specified arguments. Used inside docker container.
 Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into
-/tmp/clang-install/ directory.
+the directory specified by --to option.
 
 Available options:
   -h|--help           show this help message
-  -b|--branch         svn branch to checkout, i.e. 'trunk',
-                      'branches/release_40'
-                      (default: 'trunk')
-  -r|--revision       svn revision to checkout
-  -c|--cherrypick     revision to cherry-pick. Can be specified multiple times.
-                      Cherry-picks are performed in the sorted order using the
-                      following command:
-                      'svn patch <(svn diff -c \$rev)'.
-  -p|--llvm-project   name of an svn project to checkout. Will also add the
-                      project to a list LLVM_ENABLE_PROJECTS, passed to CMake.
-                      For clang, please use 'clang', not 'cfe'.
-                      Project 'llvm' is always included and ignored, if
-                      specified.
-                      Can be specified multiple times.
   -i|--install-target name of a cmake install target to build and include in
                       the resulting archive. Can be specified multiple times.
-Required options: At least one --install-target.
+  --to                destination directory where to install the targets.
+Required options: --to, at least one --install-target.
 
 All options after '--' are passed to CMake invocation.
 EOF
 }
 
-LLVM_SVN_REV=""
-CHERRYPICKS=""
-LLVM_BRANCH=""
 CMAKE_ARGS=""
 CMAKE_INSTALL_TARGETS=""
-# We always checkout llvm
-LLVM_PROJECTS="llvm"
-CMAKE_LLVM_ENABLE_PROJECTS=""
-CLANG_TOOLS_EXTRA_ENABLED=0
-
-function contains_project() {
-  local TARGET_PROJ="$1"
-  local PROJ
-  for PROJ in $LLVM_PROJECTS; do
-    if [ "$PROJ" == "$TARGET_PROJ" ]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-function append_project() {
-  local PROJ="$1"
-
-  LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
-  if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then
-    CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;$PROJ"
-  else
-    CMAKE_LLVM_ENABLE_PROJECTS="$PROJ"
-  fi
-}
+CLANG_INSTALL_DIR=""
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
-    -r|--revision)
-      shift
-      LLVM_SVN_REV="$1"
-      shift
-      ;;
-    -c|--cherrypick)
-      shift
-      CHERRYPICKS="$CHERRYPICKS $1"
-      shift
-      ;;
-    -b|--branch)
+    -i|--install-target)
       shift
-      LLVM_BRANCH="$1"
+      CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1"
       shift
       ;;
-    -p|--llvm-project)
+    --to)
       shift
-      PROJ="$1"
-      shift
-
-      if [ "$PROJ" == "cfe" ]; then
-        PROJ="clang"
-      fi
-
-      if [ "$PROJ" == "clang-tools-extra" ]; then
-        if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
-          echo "Project 'clang-tools-extra' is already enabled, ignoring extra occurrences."
-        else
-          CLANG_TOOLS_EXTRA_ENABLED=1
-        fi
-
-        continue
-      fi
-
-      if ! contains_project "$PROJ" ; then
-        append_project "$PROJ"
-      else
-        echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
-      fi
-      ;;
-    -i|--install-target)
-      shift
-      CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1"
+      CLANG_INSTALL_DIR="$1"
       shift
       ;;
     --)
@@ -142,102 +65,22 @@ if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then
   exit 1
 fi
 
-if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
-  if ! contains_project "clang"; then
-    echo "Project 'clang-tools-extra' was enabled without 'clang'."
-    echo "Adding 'clang' to a list of projects."
-
-    append_project "clang"
-  fi
-fi
-
-if [ "$LLVM_BRANCH" == "" ]; then
-  LLVM_BRANCH="trunk"
-fi
-
-if [ "$LLVM_SVN_REV" != "" ]; then
-  SVN_REV_ARG="-r$LLVM_SVN_REV"
-  echo "Checking out svn revision r$LLVM_SVN_REV."
-else
-  SVN_REV_ARG=""
-  echo "Checking out latest svn revision."
+if [ "$CLANG_INSTALL_DIR" == "" ]; then
+  echo "No install directory. Please specify the --to argument."
+  exit 1
 fi
 
-# Sort cherrypicks and remove duplicates.
-CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)"
-
-function apply_cherrypicks() {
-  local CHECKOUT_DIR="$1"
-
-  [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks"
-  pushd "$CHECKOUT_DIR"
-
-  # This function is always called on a sorted list of cherrypicks.
-  for CHERRY_REV in $CHERRYPICKS; do
-    echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR"
-
-    local PATCH_FILE="$(mktemp)"
-    svn diff -c $CHERRY_REV > "$PATCH_FILE"
-    svn patch "$PATCH_FILE"
-    rm "$PATCH_FILE"
-  done
-
-  popd
-}
-
 CLANG_BUILD_DIR=/tmp/clang-build
-CLANG_INSTALL_DIR=/tmp/clang-install
-
-mkdir "$CLANG_BUILD_DIR"
-
-# Get the sources from svn.
-echo "Checking out sources from svn"
-mkdir "$CLANG_BUILD_DIR/src"
-for LLVM_PROJECT in $LLVM_PROJECTS; do
-  if [ "$LLVM_PROJECT" == "clang" ]; then
-    SVN_PROJECT="cfe"
-  else
-    SVN_PROJECT="$LLVM_PROJECT"
-  fi
-
-  echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CLANG_BUILD_DIR/src/$LLVM_PROJECT"
-  svn co -q $SVN_REV_ARG \
-    "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
-    "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
-
-  # We apply cherrypicks to all repositories regardless of whether the revision
-  # changes this repository or not. For repositories not affected by the
-  # cherrypick, applying the cherrypick is a no-op.
-  apply_cherrypicks "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
-done
-
-if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
-  echo "Checking out https://llvm.org/svn/llvm-project/clang-tools-extra to $CLANG_BUILD_DIR/src/clang/tools/extra"
-  svn co -q $SVN_REV_ARG \
-    "https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \
-    "$CLANG_BUILD_DIR/src/clang/tools/extra"
 
-  apply_cherrypicks "$CLANG_BUILD_DIR/src/clang/tools/extra"
-fi
-
-CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
-
-if [ -f "$CHECKSUMS_FILE" ]; then
-  echo "Validating checksums for LLVM checkout..."
-  python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
-    --partial --multi_dir "$CLANG_BUILD_DIR/src"
-else
-  echo "Skipping checksumming checks..."
-fi
+mkdir -p "$CLANG_INSTALL_DIR"
 
-mkdir "$CLANG_BUILD_DIR/build"
+mkdir -p "$CLANG_BUILD_DIR/build"
 pushd "$CLANG_BUILD_DIR/build"
 
 # Run the build as specified in the build arguments.
 echo "Running build"
 cmake -GNinja \
   -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \
-  -DLLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS" \
   $CMAKE_ARGS \
   "$CLANG_BUILD_DIR/src/llvm"
 ninja $CMAKE_INSTALL_TARGETS
@@ -245,6 +88,6 @@ ninja $CMAKE_INSTALL_TARGETS
 popd
 
 # Cleanup.
-rm -rf "$CLANG_BUILD_DIR"
+rm -rf "$CLANG_BUILD_DIR/build"
 
 echo "Done"

utils/docker/scripts/checkout.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+#===- llvm/utils/docker/scripts/checkout.sh ---------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===-----------------------------------------------------------------------===//
+
+set -e
+
+function show_usage() {
+  cat << EOF
+Usage: checkout.sh [options]
+
+Checkout svn sources into /tmp/clang-build/src. Used inside a docker container.
+
+Available options:
+  -h|--help           show this help message
+  -b|--branch         svn branch to checkout, i.e. 'trunk',
+                      'branches/release_40'
+                      (default: 'trunk')
+  -r|--revision       svn revision to checkout
+  -c|--cherrypick     revision to cherry-pick. Can be specified multiple times.
+                      Cherry-picks are performed in the sorted order using the
+                      following command:
+                      'svn patch <(svn diff -c \$rev)'.
+  -p|--llvm-project   name of an svn project to checkout.
+                      For clang, please use 'clang', not 'cfe'.
+                      Project 'llvm' is always included and ignored, if
+                      specified.
+                      Can be specified multiple times.
+EOF
+}
+
+LLVM_SVN_REV=""
+CHERRYPICKS=""
+LLVM_BRANCH=""
+# We always checkout llvm
+LLVM_PROJECTS="llvm"
+
+function contains_project() {
+  local TARGET_PROJ="$1"
+  local PROJ
+  for PROJ in $LLVM_PROJECTS; do
+    if [ "$PROJ" == "$TARGET_PROJ" ]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -r|--revision)
+      shift
+      LLVM_SVN_REV="$1"
+      shift
+      ;;
+    -c|--cherrypick)
+      shift
+      CHERRYPICKS="$CHERRYPICKS $1"
+      shift
+      ;;
+    -b|--branch)
+      shift
+      LLVM_BRANCH="$1"
+      shift
+      ;;
+    -p|--llvm-project)
+      shift
+      PROJ="$1"
+      shift
+
+      if [ "$PROJ" == "cfe" ]; then
+        PROJ="clang"
+      fi
+
+      if ! contains_project "$PROJ" ; then
+        if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then
+          echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first."
+          LLVM_PROJECTS="$LLVM_PROJECTS clang"
+        fi
+        LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
+      else
+        echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
+      fi
+      ;;
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+  esac
+done
+
+if [ "$LLVM_BRANCH" == "" ]; then
+  LLVM_BRANCH="trunk"
+fi
+
+if [ "$LLVM_SVN_REV" != "" ]; then
+  SVN_REV_ARG="-r$LLVM_SVN_REV"
+  echo "Checking out svn revision r$LLVM_SVN_REV."
+else
+  SVN_REV_ARG=""
+  echo "Checking out latest svn revision."
+fi
+
+# Sort cherrypicks and remove duplicates.
+CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)"
+
+function apply_cherrypicks() {
+  local CHECKOUT_DIR="$1"
+
+  [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks"
+  pushd "$CHECKOUT_DIR"
+
+  # This function is always called on a sorted list of cherrypicks.
+  for CHERRY_REV in $CHERRYPICKS; do
+    echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR"
+
+    local PATCH_FILE="$(mktemp)"
+    svn diff -c $CHERRY_REV > "$PATCH_FILE"
+    svn patch "$PATCH_FILE"
+    rm "$PATCH_FILE"
+  done
+
+  popd
+}
+
+CLANG_BUILD_DIR=/tmp/clang-build
+
+# Get the sources from svn.
+echo "Checking out sources from svn"
+mkdir -p "$CLANG_BUILD_DIR/src"
+for LLVM_PROJECT in $LLVM_PROJECTS; do
+  if [ "$LLVM_PROJECT" == "clang" ]; then
+    SVN_PROJECT="cfe"
+  else
+    SVN_PROJECT="$LLVM_PROJECT"
+  fi
+
+  if [ "$SVN_PROJECT" != "clang-tools-extra" ]; then
+    CHECKOUT_DIR="$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
+  else
+    CHECKOUT_DIR="$CLANG_BUILD_DIR/src/clang/tools/extra"
+  fi
+
+  echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CHECKOUT_DIR"
+  svn co -q $SVN_REV_ARG \
+    "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
+    "$CHECKOUT_DIR"
+
+  # We apply cherrypicks to all repositories regardless of whether the revision
+  # changes this repository or not. For repositories not affected by the
+  # cherrypick, applying the cherrypick is a no-op.
+  apply_cherrypicks "$CHECKOUT_DIR"
+done
+
+CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
+
+if [ -f "$CHECKSUMS_FILE" ]; then
+  echo "Validating checksums for LLVM checkout..."
+  python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
+    --partial --multi_dir "$CLANG_BUILD_DIR/src"
+else
+  echo "Skipping checksumming checks..."
+fi
+
+echo "Done"