Reland "depot_tools: Bootstrap Python 3 on Linux/Mac"

Don't call ensure_bootstrap from update_depot_tools.
ensure_bootstrap also updates gsutil and all versions of pylint
which is slow, particularly on MinGW.

Original change's description:
> depot_tools: Bootstrap Python 3 on Linux/Mac
>
> This will make it possible for developers to execute depot_tools
> scripts using Python 3 in a known environment.
>
> Bug: 1002153
> Change-Id: I5ff492a49d227c1b5876f49adba020f51a575bdd
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1762664
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
> Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> Reviewed-by: Andrii Shyshkalov <tandrii@google.com>

Bug: 1002153
Change-Id: Ia7579e440438897ba4a7c65a8b228dcfe7f28c86
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1810040
Auto-Submit: Edward Lesmes <ehmaldonado@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

.gitignore

@@ -33,6 +33,7 @@
 /.subversion
 
 # Ignore locations where third-party tools are placed during bootstrapping.
+/bootstrap*_bin
 /python*_bin
 /python_bin_reldir.txt
 /python3_bin_reldir.txt

bootstrap/win/win_tools.py → bootstrap/bootstrap.py

@@ -19,15 +19,16 @@ import tempfile
 
 
 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
-ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
+ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..'))
 
 DEVNULL = open(os.devnull, 'w')
 
-BAT_EXT = '.bat' if sys.platform.startswith('win') else ''
+IS_WIN = sys.platform.startswith('win')
+BAT_EXT = '.bat' if IS_WIN else ''
 
 # Top-level stubs to generate that fall through to executables within the Git
 # directory.
-STUBS = {
+WIN_GIT_STUBS = {
   'git.bat': 'cmd\\git.exe',
   'gitk.bat': 'cmd\\gitk.exe',
   'ssh.bat': 'usr\\bin\\ssh.exe',
@@ -88,6 +89,7 @@ def maybe_update(content, dst_path):
   logging.debug('Updating %r', dst_path)
   with open(dst_path, 'w') as fd:
     fd.write(content)
+  os.chmod(dst_path, 0o755)
   return True
 
 
@@ -200,7 +202,7 @@ def _safe_rmtree(path):
 
   def _make_writable_and_remove(path):
     st = os.stat(path)
-    new_mode = st.st_mode | 0200
+    new_mode = st.st_mode | 0o200
     if st.st_mode == new_mode:
       return False
     try:
@@ -226,7 +228,7 @@ def clean_up_old_installations(skip_dir):
   that is using the bootstrapped Python!
   """
   root_contents = os.listdir(ROOT_DIR)
-  for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin'):
+  for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin', 'bootstrap-*_bin'):
     for entry in fnmatch.filter(root_contents, f):
       full_entry = os.path.join(ROOT_DIR, entry)
       if full_entry == skip_dir or not os.path.isdir(full_entry):
@@ -266,7 +268,7 @@ def git_postprocess(template, git_directory):
     logging.info('Could not find mingw directory for %r.', git_directory)
 
   # Create Git templates and configure its base layout.
-  for stub_name, relpath in STUBS.iteritems():
+  for stub_name, relpath in WIN_GIT_STUBS.items():
     stub_template = template._replace(GIT_PROGRAM=relpath)
     stub_template.maybe_install(
         'git.template.bat',
@@ -293,7 +295,7 @@ def git_postprocess(template, git_directory):
 def main(argv):
   parser = argparse.ArgumentParser()
   parser.add_argument('--verbose', action='store_true')
-  parser.add_argument('--win-tools-name', required=True,
+  parser.add_argument('--bootstrap-name', required=True,
                       help='The directory of the Python installation.')
   parser.add_argument('--bleeding-edge', action='store_true',
                       help='Force bleeding edge Git.')
@@ -302,52 +304,59 @@ def main(argv):
   logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
 
   template = Template.empty()._replace(
-      PYTHON_RELDIR=os.path.join(args.win_tools_name, 'python'),
-      PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'),
+      PYTHON_RELDIR=os.path.join(args.bootstrap_name, 'python'),
+      PYTHON_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python', 'bin'),
       PYTHON_BIN_RELDIR_UNIX=posixpath.join(
-          args.win_tools_name, 'python', 'bin'),
-      PYTHON3_BIN_RELDIR=os.path.join(args.win_tools_name, 'python3', 'bin'),
+          args.bootstrap_name, 'python', 'bin'),
+      PYTHON3_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python3', 'bin'),
       PYTHON3_BIN_RELDIR_UNIX=posixpath.join(
-          args.win_tools_name, 'python3', 'bin'),
-      GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'),
-      GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git'))
+          args.bootstrap_name, 'python3', 'bin'),
+      GIT_BIN_RELDIR=os.path.join(args.bootstrap_name, 'git'),
+      GIT_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'git'))
 
-  win_tools_dir = os.path.join(ROOT_DIR, args.win_tools_name)
-  git_postprocess(template, os.path.join(win_tools_dir, 'git'))
+  bootstrap_dir = os.path.join(ROOT_DIR, args.bootstrap_name)
 
   # Clean up any old Python and Git installations.
-  clean_up_old_installations(win_tools_dir)
-
-  # Emit our Python bin depot-tools-relative directory. This is ready by
-  # "python.bat" to identify the path of the current Python installation.
-  #
-  # We use this indirection so that upgrades can change this pointer to
-  # redirect "python.bat" to a new Python installation. We can't just update
-  # "python.bat" because batch file executions reload the batch file and seek
-  # to the previous cursor in between every command, so changing the batch
-  # file contents could invalidate any existing executions.
-  #
-  # The intention is that the batch file itself never needs to change when
-  # switching Python versions.
-  maybe_update(
-      template.PYTHON_BIN_RELDIR,
-      os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
+  clean_up_old_installations(bootstrap_dir)
+
+  # Only bootstrap git and Python 2 on Windows.
+  if IS_WIN:
+    git_postprocess(template, os.path.join(bootstrap_dir, 'git'))
+
+    # Emit our Python bin depot-tools-relative directory. This is ready by
+    # "python.bat" to identify the path of the current Python installation.
+    #
+    # We use this indirection so that upgrades can change this pointer to
+    # redirect "python.bat" to a new Python installation. We can't just update
+    # "python.bat" because batch file executions reload the batch file and seek
+    # to the previous cursor in between every command, so changing the batch
+    # file contents could invalidate any existing executions.
+    #
+    # The intention is that the batch file itself never needs to change when
+    # switching Python versions.
+    maybe_update(
+        template.PYTHON_BIN_RELDIR,
+        os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
+
+    python_template = 'python27.%s.bat' % (
+        'bleeding_edge' if args.bleeding_edge else 'new')
+    for src_name, dst_name in (
+        ('git-bash.template.sh', 'git-bash'),
+        (python_template, 'python' + BAT_EXT),
+        ):
+      # Re-evaluate and regenerate our root templated files.
+      template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
+
   maybe_update(
       template.PYTHON3_BIN_RELDIR,
       os.path.join(ROOT_DIR, 'python3_bin_reldir.txt'))
 
-  python_bat_template = ('python27.new.bat' if not args.bleeding_edge
-                         else 'python27.bleeding_edge.bat')
-  python3_bat_template = ('python3.new.bat' if not args.bleeding_edge
-                          else 'python3.bleeding_edge.bat')
+  python3_template = 'python3.'
+  python3_template += 'bleeding_edge' if args.bleeding_edge else 'new'
+  python3_template += BAT_EXT
 
-  # Re-evaluate and regenerate our root templated files.
-  for src_name, dst_name in (
-      ('git-bash.template.sh', 'git-bash'),
-      (python_bat_template, 'python.bat'),
-      (python3_bat_template, 'python3.bat'),
-      ):
-    template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
+  template.maybe_install(
+      python3_template, os.path.join(ROOT_DIR, 'python3' + BAT_EXT))
 
   return 0
 

bootstrap/manifest.txt

@@ -0,0 +1,27 @@
+# CIPD manifest for bootstrapping tools.
+#
+# We must install anything that we want included on PATH to a different
+# subdirectory than Git, as Git's msys bash strips its root directory
+# from PATH, hence the subdirs.
+#
+# If any paths or package layouts change, updates will be required in
+# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates.
+#
+# "win_tools.bat" has a hard requirement that the Python packages contain the
+# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with
+# the CIPD tag "version:VERSION". It uses this to extract VERSION.
+#
+# "bootstrap_python3" has a hard requirement that the Python package contains
+# the string "cpython3/" and ends with the CIPD tag "version:VERSION".
+# It uses this to extract VERSION.
+
+$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64
+
+@Subdir python
+infra/python/cpython/${platform} version:2.7.15.chromium14
+
+@Subdir python3
+infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
+
+@Subdir git
+infra/git/${os}-${arch} version:2.21.0.chromium16

bootstrap/manifest_bleeding_edge.txt

@@ -0,0 +1,27 @@
+# CIPD manifest for bootstrapping tools.
+#
+# We must install anything that we want included on PATH to a different
+# subdirectory than Git, as Git's msys bash strips its root directory
+# from PATH, hence the subdirs.
+#
+# If any paths or package layouts change, updates will be required in
+# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates.
+#
+# "win_tools.bat" has a hard requirement that the Python packages contain the
+# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with
+# the CIPD tag "version:VERSION". It uses this to extract VERSION.
+#
+# "bootstrap_python3" has a hard requirement that the Python package contains
+# the string "cpython3/" and ends with the CIPD tag "version:VERSION".
+# It uses this to extract VERSION.
+
+$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64
+
+@Subdir python
+infra/python/cpython/${platform} version:2.7.15.chromium14
+
+@Subdir python3
+infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
+
+@Subdir git
+infra/git/${os}-${arch} version:2.21.0.chromium16

bootstrap/python3.bleeding_edge

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+base_dir=$(dirname "$0")
+
+PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
+PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
+"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"

bootstrap/python3.new

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+base_dir=$(dirname "$0")
+
+PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)"
+PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH"
+"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@"

bootstrap/win/manifest.txt

@@ -1,23 +0,0 @@
-# CIPD manifest for Windows tools.
-#
-# We must install anything that we want included on PATH to a different
-# subdirectory than Git, as Git's msys bash strips its root directory
-# from PATH, hence the subdirs.
-#
-# If any paths or package layouts change, updates will be required in
-# "win_tools.bat" and "win_tools.py" templates.
-#
-# "win_tools.bat" has a hard requirement that the Python package contains the
-# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
-# to extract VERSION.
-
-$VerifiedPlatform windows-386 windows-amd64
-
-@Subdir python
-infra/python/cpython/${platform} version:2.7.15.chromium14
-
-@Subdir python3
-infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
-
-@Subdir git
-infra/git/windows-${arch} version:2.21.0.chromium16

bootstrap/win/manifest_bleeding_edge.txt

@@ -1,23 +0,0 @@
-# CIPD manifest for Windows tools.
-#
-# We must install anything that we want included on PATH to a different
-# subdirectory than Git, as Git's msys bash strips its root directory
-# from PATH, hence the subdirs.
-#
-# If any paths or package layouts change, updates will be required in
-# "win_tools.bat" and "win_tools.py" templates.
-#
-# "win_tools.bat" has a hard requirement that the Python package contains the
-# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
-# to extract VERSION.
-
-$VerifiedPlatform windows-386 windows-amd64
-
-@Subdir python
-infra/python/cpython/${platform} version:2.7.15.chromium14
-
-@Subdir python3
-infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1
-
-@Subdir git
-infra/git/windows-${arch} version:2.21.0.chromium16

bootstrap/win/win_tools.bat → bootstrap/win_tools.bat

@@ -11,19 +11,19 @@
 setlocal EnableDelayedExpansion
 
 :: Get absolute root directory (.js scripts don't handle relative paths well).
-pushd %~dp0..\..
-set WIN_TOOLS_ROOT_DIR=%CD%
+pushd %~dp0..
+set BOOTSTRAP_ROOT_DIR=%CD%
 popd
 
 :: Extra arguments to pass to our "win_tools.py" script.
-set WIN_TOOLS_EXTRA_ARGS=
+set BOOTSTRAP_EXTRA_ARGS=
 
 :: Determine if we're running a bleeding-edge installation.
-if not exist "%WIN_TOOLS_ROOT_DIR%\.bleeding_edge" (
+if not exist "%BOOTSTRAP_ROOT_DIR%\.bleeding_edge" (
   set CIPD_MANIFEST=manifest.txt
 ) else (
   set CIPD_MANIFEST=manifest_bleeding_edge.txt
-  set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --bleeding-edge
+  set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bleeding-edge
 )
 
 :: Parse our CIPD manifest and identify the "cpython" version. We do this by
@@ -51,25 +51,25 @@ if "%PYTHON3_VERSION%" == "" (
 )
 
 :: We will take the version string, replace "." with "_", and surround it with
-:: "win-tools-<PYTHON_VERSION>_bin" so that it matches "win_tools.py"'s cleanup
+:: "bootstrap-<PYTHON3_VERSION>_bin" so that it matches "win_tools.py"'s cleanup
 :: expression and ".gitignore".
 ::
-:: We incorporate PYTHON_VERSION into the "win_tools" directory name so that
+:: We incorporate PYTHON3_VERSION into the "win_tools" directory name so that
 :: new installations don't interfere with long-running Python processes if
 :: Python is upgraded.
-set WIN_TOOLS_NAME=win_tools-%PYTHON_VERSION:.=_%_bin
-set WIN_TOOLS_PATH=%WIN_TOOLS_ROOT_DIR%\%WIN_TOOLS_NAME%
-set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --win-tools-name "%WIN_TOOLS_NAME%"
+set BOOTSTRAP_NAME=bootstrap-%PYTHON3_VERSION:.=_%_bin
+set BOOTSTRAP_PATH=%BOOTSTRAP_ROOT_DIR%\%BOOTSTRAP_NAME%
+set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bootstrap-name "%BOOTSTRAP_NAME%"
 
 :: Install our CIPD packages. The CIPD client self-bootstraps.
 :: See "//cipd.bat" and "//cipd.ps1" for more information.
-set CIPD_EXE=%WIN_TOOLS_ROOT_DIR%\cipd.bat
-call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%WIN_TOOLS_PATH%"
+set CIPD_EXE=%BOOTSTRAP_ROOT_DIR%\cipd.bat
+call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%BOOTSTRAP_PATH%"
 if errorlevel 1 goto :END
 
 :: This executes "win_tools.py" using the bundle's Python interpreter.
-set WIN_TOOLS_PYTHON_BIN=%WIN_TOOLS_PATH%\python\bin\python.exe
-call "%WIN_TOOLS_PYTHON_BIN%" "%~dp0win_tools.py" %WIN_TOOLS_EXTRA_ARGS%
+set BOOTSTRAP_PYTHON_BIN=%BOOTSTRAP_PATH%\python3\bin\python3.exe
+call "%BOOTSTRAP_PYTHON_BIN%" "%~dp0bootstrap.py" %BOOTSTRAP_EXTRA_ARGS%
 
 
 :END

bootstrap_python3

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+function bootstrap_python3 {
+  base_dir=$(dirname "${BASH_SOURCE[0]}")
+
+  cd $base_dir
+
+  if [ -e ".bleeding_edge" ]; then
+    CIPD_MANIFEST="bootstrap/manifest_bleeding_edge.txt"
+  else
+    CIPD_MANIFEST="bootstrap/manifest.txt"
+  fi
+
+  while IFS= read -r line; do
+    if [[ $line =~ ^[^#].*cpython3/.*version:(.*)$ ]]; then
+      PYTHON3_VERSION=${BASH_REMATCH[1]}
+      PYTHON3_VERSION=${PYTHON3_VERSION//[[:space:]]/}
+    fi
+  done < $CIPD_MANIFEST
+  if [ "X$PYTHON3_VERSION" == "X" ]; then
+    echo Could not extract Python 3 version from manifest.
+    return 1
+  fi
+
+  BOOTSTRAP_PATH="bootstrap-${PYTHON3_VERSION}_bin"
+
+  # Install CIPD packages. The CIPD client self-bootstraps.
+  "cipd" ensure -log-level warning -ensure-file "${CIPD_MANIFEST}" \
+      -root "$BOOTSTRAP_PATH"
+
+  BOOTSTRAP_PYTHON_BIN="${BOOTSTRAP_PATH}/python3/bin/python3"
+  "$BOOTSTRAP_PYTHON_BIN" "bootstrap/bootstrap.py" --bootstrap-name $BOOTSTRAP_PATH
+
+  cd - > /dev/null
+}

ensure_bootstrap

@@ -26,6 +26,13 @@ else
   if [ -L "$base_dir" ]; then
     base_dir=`cd "$base_dir" && pwd -P`
   fi
+
+  # Don't bootstrap Python 3 on windows, since it is already done by
+  # bootstrap/win_tools.bat.
+  if [ "X$DEPOT_TOOLS_BOOTSTRAP_PYTHON3" != "X0" ]; then
+    source "$base_dir/bootstrap_python3"
+    bootstrap_python3
+  fi
 fi
 
 # Sync CIPD-boostrapped packages.

update_depot_tools

@@ -45,7 +45,7 @@ fi
 
 # We want to update the bundled tools even under MinGW.
 if [ $MINGW = 0 ]; then
-  $COMSPEC /c `cygpath -w "$base_dir/bootstrap/win/win_tools.bat"`
+  $COMSPEC /c `cygpath -w "$base_dir/bootstrap/win_tools.bat"`
   case $? in
     123)
       # msys environment was upgraded, need to quit.
@@ -118,10 +118,15 @@ if [ "X$DEPOT_TOOLS_UPDATE" != "X0" ]; then
     cd - > /dev/null
   fi
 
-  # Sync CIPD and CIPD client tools.
+  # Sync CIPD-boostrapped packages.
   source "$base_dir/cipd_bin_setup.sh"
   cipd_bin_setup
 
-  find "$base_dir" | grep -i ".*\.pyc" | xargs rm -f;
+  # Don't bootstrap Python 3 on windows, since it is already done by
+  # bootstrap/win_tools.bat.
+  if [ "X$MINGW" != "X0" -a "X$DEPOT_TOOLS_BOOTSTRAP_PYTHON3" != "X0" ]; then
+    source "$base_dir/bootstrap_python3"
+    bootstrap_python3
+  fi
 fi
 

update_depot_tools.bat

@@ -24,7 +24,7 @@ IF EXIST "%DEPOT_TOOLS_DIR%.disable_auto_update" GOTO :EOF
 set GIT_URL=https://chromium.googlesource.com/chromium/tools/depot_tools.git
 
 :: Will download git and python.
-call "%DEPOT_TOOLS_DIR%bootstrap\win\win_tools.bat"
+call "%DEPOT_TOOLS_DIR%bootstrap\win_tools.bat"
 if errorlevel 1 goto :EOF
 :: Now clear errorlevel so it can be set by other programs later.
 set errorlevel=