[siso] add autosiso wrappers

Developers will be able to use `autosiso` command to invoke siso builds.
This CL extracts reclient management logic from `ninja_reclient` to be  reused in both Ninja and Siso builds.

Bug: b/278675516
Change-Id: I3e64a3188db184f4d3f851063a0feef7d3a73d6d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4445366
Auto-Submit: Junji Watanabe <jwata@google.com>
Reviewed-by: Philipp Wollermann <philwo@google.com>
Commit-Queue: Joanna Wang <jojwang@chromium.org>
Reviewed-by: Fumitoshi Ukai <ukai@google.com>
Reviewed-by: Takuto Ikuta <tikuta@chromium.org>
Reviewed-by: Joanna Wang <jojwang@chromium.org>

OWNERS

@@ -11,12 +11,15 @@ per-file autoninja*=dpranke@google.com
 
 per-file gn*=dpranke@google.com
 
+per-file autosiso*=file://BUILD_OWNERS
 per-file ninja*=dpranke@google.com
 per-file ninja*=thakis@chromium.org
 per-file ninja_reclient.py=file://BUILD_OWNERS
 per-file ninja_reclient.py=file://RECLIENT_OWNERS
 per-file ninjalog*=tikuta@chromium.org
 per-file post_build_ninja_summary.py=brucedawson@chromium.org
+per-file reclient_helper.py=file://BUILD_OWNERS
+per-file reclient_helper.py=file://RECLIENT_OWNERS
 per-file reclientreport*=file://RECLIENT_OWNERS
 
 per-file presubmit*.py=brucedawson@chromium.org

autosiso

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+base_dir=$(dirname "$0")
+PYTHONDONTWRITEBYTECODE=1 exec python3 "$base_dir/autosiso.py" "$@"

autosiso.bat

@@ -0,0 +1,12 @@
+@echo off
+:: Copyright 2023 The Chromium Authors
+:: Use of this source code is governed by a BSD-style license that can be
+:: found in the LICENSE file.
+setlocal
+
+:: Ensure that "depot_tools" is somewhere in PATH so this tool can be used
+:: standalone, but allow other PATH manipulations to take priority.
+set PATH=%PATH%;%~dp0
+
+:: Defer control.
+python3 "%~dp0\autosiso.py" "%*"

autosiso.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# Copyright 2023 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""
+Developers invoke this script via autosiso or autosiso.bat to simply run
+Siso builds.
+"""
+
+import sys
+
+import reclient_helper
+import siso
+
+
+def main(argv):
+  with reclient_helper.build_context(argv) as ret_code:
+    if ret_code:
+      return ret_code
+    argv = [
+        argv[0],
+        'ninja',
+        # Do not authenticate when using Reproxy.
+        '-project=',
+        '-reapi_instance=',
+    ] + argv[1:]
+    return siso.main(argv)
+
+
+if __name__ == '__main__':
+  try:
+    sys.exit(main(sys.argv))
+  except KeyboardInterrupt:
+    sys.exit(1)

ninja_reclient.py

@@ -7,144 +7,20 @@ handles the client lifecycle safely. It will automatically start
 reproxy before running ninja and stop reproxy when ninja stops
 for any reason eg. build completes, keyboard interupt etc."""
 
-import hashlib
-import os
-import subprocess
 import sys
 
 import ninja
-import gclient_paths
-
-
-def find_reclient_bin_dir():
-  tools_path = gclient_paths.GetBuildtoolsPath()
-  if not tools_path:
-    return None
-
-  reclient_bin_dir = os.path.join(tools_path, 'reclient')
-  if os.path.isdir(reclient_bin_dir):
-    return reclient_bin_dir
-  return None
-
-
-def find_reclient_cfg():
-  tools_path = gclient_paths.GetBuildtoolsPath()
-  if not tools_path:
-    return None
-
-  reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg')
-  if os.path.isfile(reclient_cfg):
-    return reclient_cfg
-  return None
-
-
-def run(cmd_args):
-  if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1':
-    print(' '.join(cmd_args))
-  return subprocess.call(cmd_args)
-
-
-def start_reproxy(reclient_cfg, reclient_bin_dir):
-  return run([
-      os.path.join(reclient_bin_dir, 'bootstrap'),
-      '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy'),
-      '--cfg=' + reclient_cfg
-  ])
-
-
-def stop_reproxy(reclient_cfg, reclient_bin_dir):
-  return run([
-      os.path.join(reclient_bin_dir, 'bootstrap'), '--shutdown',
-      '--cfg=' + reclient_cfg
-  ])
-
-
-def find_rel_ninja_out_dir(args):
-  # Ninja uses getopt_long, which allow to intermix non-option arguments.
-  # To leave non supported parameters untouched, we do not use getopt.
-  for index, arg in enumerate(args[1:]):
-    if arg == '-C':
-      # + 1 to get the next argument and +1 because we trimmed off args[0]
-      return args[index + 2]
-    if arg.startswith('-C'):
-      # Support -Cout/Default
-      return arg[2:]
-  return '.'
-
-
-def set_reproxy_path_flags(out_dir, make_dirs=True):
-  """Helper to setup the logs and cache directories for reclient
-
-  Creates the following directory structure if make_dirs is true:
-  out_dir/
-    .reproxy_tmp/
-      logs/
-      cache/
-
-  The following env vars are set if not already set:
-    RBE_output_dir=out_dir/.reproxy_tmp/logs
-    RBE_proxy_log_dir=out_dir/.reproxy_tmp/logs
-    RBE_log_dir=out_dir/.reproxy_tmp/logs
-    RBE_cache_dir=out_dir/.reproxy_tmp/cache
-  *Nix Only:
-    RBE_server_address=unix://out_dir/.reproxy_tmp/reproxy.sock
-  Windows Only:
-    RBE_server_address=pipe://md5(out_dir/.reproxy_tmp)/reproxy.pipe
-  """
-  tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp'))
-  log_dir = os.path.join(tmp_dir, 'logs')
-  cache_dir = os.path.join(tmp_dir, 'cache')
-  if make_dirs:
-    os.makedirs(tmp_dir, exist_ok=True)
-    os.makedirs(log_dir, exist_ok=True)
-    os.makedirs(cache_dir, exist_ok=True)
-  os.environ.setdefault("RBE_output_dir", log_dir)
-  os.environ.setdefault("RBE_proxy_log_dir", log_dir)
-  os.environ.setdefault("RBE_log_dir", log_dir)
-  os.environ.setdefault("RBE_cache_dir", cache_dir)
-  if sys.platform.startswith('win'):
-    pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest()
-    os.environ.setdefault("RBE_server_address",
-                          "pipe://%s/reproxy.pipe" % pipe_dir)
-  else:
-    os.environ.setdefault("RBE_server_address",
-                          "unix://%s/reproxy.sock" % tmp_dir)
+import reclient_helper
 
 
 def main(argv):
-  # If use_remoteexec is set, but the reclient binaries or configs don't
-  # exist, display an error message and stop.  Otherwise, the build will
-  # attempt to run with rewrapper wrapping actions, but will fail with
-  # possible non-obvious problems.
-  # As of January 2023, dev builds with reclient are not supported, so
-  # indicate that use_goma should be swapped for use_remoteexec.  This
-  # message will be changed when dev builds are fully supported.
-  reclient_bin_dir = find_reclient_bin_dir()
-  reclient_cfg = find_reclient_cfg()
-  if reclient_bin_dir is None or reclient_cfg is None:
-    print(("Build is configured to use reclient but necessary binaries "
-           "or config files can't be found.  Developer builds with "
-           "reclient are not yet supported.  Try regenerating your "
-           "build with use_goma in place of use_remoteexec for now."),
-          file=sys.stderr)
-    return 1
-  try:
-    set_reproxy_path_flags(find_rel_ninja_out_dir(argv))
-  except OSError:
-    print("Error creating reproxy_tmp in output dir", file=sys.stderr)
-    return 1
-  reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir)
-  if reproxy_ret_code != 0:
-    return reproxy_ret_code
-  try:
-    return ninja.main(argv)
-  except KeyboardInterrupt:
-    print("Caught User Interrupt", file=sys.stderr)
-    # Suppress python stack trace if ninja is interrupted
-    return 1
-  finally:
-    print("Shutting down reproxy...", file=sys.stderr)
-    stop_reproxy(reclient_cfg, reclient_bin_dir)
+  with reclient_helper.build_context(argv) as ret_code:
+    if ret_code:
+      return ret_code
+    try:
+      return ninja.main(argv)
+    except KeyboardInterrupt:
+      return 1
 
 
 if __name__ == '__main__':

reclient_helper.py

@@ -0,0 +1,143 @@
+# Copyright 2023 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""This helper provides a build context that handles
+the reclient lifecycle safely. It will automatically start
+reproxy before running ninja and stop reproxy when build stops
+for any reason e.g. build completion, keyboard interrupt etc."""
+
+import contextlib
+import hashlib
+import os
+import subprocess
+import sys
+
+import gclient_paths
+
+
+def find_reclient_bin_dir():
+  tools_path = gclient_paths.GetBuildtoolsPath()
+  if not tools_path:
+    return None
+
+  reclient_bin_dir = os.path.join(tools_path, 'reclient')
+  if os.path.isdir(reclient_bin_dir):
+    return reclient_bin_dir
+  return None
+
+
+def find_reclient_cfg():
+  tools_path = gclient_paths.GetBuildtoolsPath()
+  if not tools_path:
+    return None
+
+  reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg')
+  if os.path.isfile(reclient_cfg):
+    return reclient_cfg
+  return None
+
+
+def run(cmd_args):
+  if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1':
+    print(' '.join(cmd_args))
+  return subprocess.call(cmd_args)
+
+
+def start_reproxy(reclient_cfg, reclient_bin_dir):
+  return run([
+      os.path.join(reclient_bin_dir, 'bootstrap'),
+      '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy'),
+      '--cfg=' + reclient_cfg
+  ])
+
+
+def stop_reproxy(reclient_cfg, reclient_bin_dir):
+  return run([
+      os.path.join(reclient_bin_dir, 'bootstrap'), '--shutdown',
+      '--cfg=' + reclient_cfg
+  ])
+
+
+def find_ninja_out_dir(args):
+  # Ninja uses getopt_long, which allows to intermix non-option arguments.
+  # To leave non supported parameters untouched, we do not use getopt.
+  for index, arg in enumerate(args[1:]):
+    if arg == '-C':
+      # + 1 to get the next argument and +1 because we trimmed off args[0]
+      return args[index + 2]
+    if arg.startswith('-C'):
+      # Support -Cout/Default
+      return arg[2:]
+  return '.'
+
+
+def set_reproxy_path_flags(out_dir, make_dirs=True):
+  """Helper to setup the logs and cache directories for reclient.
+
+  Creates the following directory structure if make_dirs is true:
+  out_dir/
+    .reproxy_tmp/
+      logs/
+      cache/
+
+  The following env vars are set if not already set:
+    RBE_output_dir=out_dir/.reproxy_tmp/logs
+    RBE_proxy_log_dir=out_dir/.reproxy_tmp/logs
+    RBE_log_dir=out_dir/.reproxy_tmp/logs
+    RBE_cache_dir=out_dir/.reproxy_tmp/cache
+  *Nix Only:
+    RBE_server_address=unix://out_dir/.reproxy_tmp/reproxy.sock
+  Windows Only:
+    RBE_server_address=pipe://md5(out_dir/.reproxy_tmp)/reproxy.pipe
+  """
+  tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp'))
+  log_dir = os.path.join(tmp_dir, 'logs')
+  cache_dir = os.path.join(tmp_dir, 'cache')
+  if make_dirs:
+    os.makedirs(tmp_dir, exist_ok=True)
+    os.makedirs(log_dir, exist_ok=True)
+    os.makedirs(cache_dir, exist_ok=True)
+  os.environ.setdefault("RBE_output_dir", log_dir)
+  os.environ.setdefault("RBE_proxy_log_dir", log_dir)
+  os.environ.setdefault("RBE_log_dir", log_dir)
+  os.environ.setdefault("RBE_cache_dir", cache_dir)
+  if sys.platform.startswith('win'):
+    pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest()
+    os.environ.setdefault("RBE_server_address",
+                          "pipe://%s/reproxy.pipe" % pipe_dir)
+  else:
+    os.environ.setdefault("RBE_server_address",
+                          "unix://%s/reproxy.sock" % tmp_dir)
+
+
+@contextlib.contextmanager
+def build_context(argv):
+  # If use_remoteexec is set, but the reclient binaries or configs don't
+  # exist, display an error message and stop.  Otherwise, the build will
+  # attempt to run with rewrapper wrapping actions, but will fail with
+  # possible non-obvious problems.
+  reclient_bin_dir = find_reclient_bin_dir()
+  reclient_cfg = find_reclient_cfg()
+  if reclient_bin_dir is None or reclient_cfg is None:
+    print(("Build is configured to use reclient but necessary binaries "
+           "or config files can't be found.  Developer builds with "
+           "reclient are not yet supported.  Try regenerating your "
+           "build with use_goma in place of use_remoteexec for now."),
+          file=sys.stderr)
+    yield 1
+    return
+  try:
+    set_reproxy_path_flags(find_ninja_out_dir(argv))
+  except OSError:
+    print("Error creating reproxy_tmp in output dir", file=sys.stderr)
+    yield 1
+    return
+  reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir)
+  if reproxy_ret_code != 0:
+    yield reproxy_ret_code
+    return
+  try:
+    yield
+  finally:
+    print("Shutting down reproxy...", file=sys.stderr)
+    stop_reproxy(reclient_cfg, reclient_bin_dir)