autoninja: add simple test

This is to prevent revert like https://crrev.com/c/3607513

Bug: 1317620
Change-Id: I6ab7aba5f92719bd573d22d90358f58e48aeb10c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3607514
Reviewed-by: Bruce Dawson <brucedawson@chromium.org>
Reviewed-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Takuto Ikuta <tikuta@chromium.org>

PRESUBMIT.py

@@ -124,7 +124,10 @@ def CheckUnitTestsOnCommit(input_api, output_api):
       'recipes_test.py',
   ]
 
-  py3_only_tests = ['ninjalog_uploader_test.py']
+  py3_only_tests = [
+      'autoninja_test.py',
+      'ninjalog_uploader_test.py',
+  ]
 
   tests = input_api.canned_checks.GetUnitTestsInDirectory(
       input_api,

autoninja.py

@@ -21,198 +21,207 @@ import sys
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
-# The -t tools are incompatible with -j
-t_specified = False
-j_specified = False
-offline = False
-output_dir = '.'
-input_args = sys.argv
-# On Windows the autoninja.bat script passes along the arguments enclosed in
-# double quotes. This prevents multiple levels of parsing of the special '^'
-# characters needed when compiling a single file but means that this script gets
-# called with a single argument containing all of the actual arguments,
-# separated by spaces. When this case is detected we need to do argument
-# splitting ourselves. This means that arguments containing actual spaces are
-# not supported by autoninja, but that is not a real limitation.
-if (sys.platform.startswith('win') and len(sys.argv) == 2 and
-    input_args[1].count(' ') > 0):
-  input_args = sys.argv[:1] + sys.argv[1].split()
-
-# Ninja uses getopt_long, which allow to intermix non-option arguments.
-# To leave non supported parameters untouched, we do not use getopt.
-for index, arg in enumerate(input_args[1:]):
-  if arg.startswith('-j'):
-    j_specified = True
-  if arg.startswith('-t'):
-    t_specified = True
-  if arg == '-C':
-    # + 1 to get the next argument and +1 because we trimmed off input_args[0]
-    output_dir = input_args[index + 2]
-  elif arg.startswith('-C'):
-    # Support -Cout/Default
-    output_dir = arg[2:]
-  elif arg in ('-o', '--offline'):
-    offline = True
-  elif arg == '-h':
-    print('autoninja: Use -o/--offline to temporary disable goma.',
-          file=sys.stderr)
-    print(file=sys.stderr)
-
-# Strip -o/--offline so ninja doesn't see them.
-input_args = [ arg for arg in input_args if arg not in ('-o', '--offline')]
-
-use_goma = False
-use_remoteexec = False
-
-# Currently get reclient binary and config dirs relative to output_dir.  If
-# they exist and using remoteexec, then automatically call bootstrap to start
-# reproxy.  This works under the current assumption that the output
-# directory is two levels up from chromium/src.
-reclient_bin_dir = os.path.join(
-  output_dir, '..', '..', 'buildtools', 'reclient')
-reclient_cfg = os.path.join(
-  output_dir, '..', '..', 'buildtools', 'reclient_cfgs', 'reproxy.cfg')
-
-# Attempt to auto-detect remote build acceleration. We support gn-based
-# builds, where we look for args.gn in the build tree, and cmake-based builds
-# where we look for rules.ninja.
-if os.path.exists(os.path.join(output_dir, 'args.gn')):
-  with open(os.path.join(output_dir, 'args.gn')) as file_handle:
-    for line in file_handle:
-      # Either use_goma, use_remoteexec or use_rbe (in deprecation)
-      # activate build acceleration.
-      #
-      # This test can match multi-argument lines. Examples of this are:
-      # is_debug=false use_goma=true is_official_build=false
-      # use_goma=false# use_goma=true This comment is ignored
-      #
-      # Anything after a comment is not consider a valid argument.
-      line_without_comment = line.split('#')[0]
-      if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
-                   line_without_comment):
-        use_goma = True
-        continue
-      if re.search(r'(^|\s)(use_rbe|use_remoteexec)\s*=\s*true($|\s)',
-                   line_without_comment):
-        use_remoteexec = True
-        continue
-else:
-  for relative_path in [
-      '',  # GN keeps them in the root of output_dir
-      'CMakeFiles'
-  ]:
-    path = os.path.join(output_dir, relative_path, 'rules.ninja')
-    if os.path.exists(path):
-      with open(path) as file_handle:
-        for line in file_handle:
-          if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
-            use_goma = True
-            break
-
-# If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" (case-insensitive)
-# then gomacc will use the local compiler instead of doing a goma compile. This
-# is convenient if you want to briefly disable goma. It avoids having to rebuild
-# the world when transitioning between goma/non-goma builds. However, it is not
-# as fast as doing a "normal" non-goma build because an extra process is created
-# for each compile step. Checking this environment variable ensures that
-# autoninja uses an appropriate -j value in this situation.
-goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
-if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
-  use_goma = False
 
-if use_goma:
-  gomacc_file = 'gomacc.exe' if sys.platform.startswith('win') else 'gomacc'
-  goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin'))
-  gomacc_path = os.path.join(goma_dir, gomacc_file)
-  # Don't invoke gomacc if it doesn't exist.
-  if os.path.exists(gomacc_path):
-    # Check to make sure that goma is running. If not, don't start the build.
-    status = subprocess.call([gomacc_path, 'port'], stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE, shell=False)
-    if status == 1:
-      print('Goma is not running. Use "goma_ctl ensure_start" to start it.',
+def main(args):
+  # The -t tools are incompatible with -j
+  t_specified = False
+  j_specified = False
+  offline = False
+  output_dir = '.'
+  input_args = args
+  # On Windows the autoninja.bat script passes along the arguments enclosed in
+  # double quotes. This prevents multiple levels of parsing of the special '^'
+  # characters needed when compiling a single file but means that this script
+  # gets called with a single argument containing all of the actual arguments,
+  # separated by spaces. When this case is detected we need to do argument
+  # splitting ourselves. This means that arguments containing actual spaces are
+  # not supported by autoninja, but that is not a real limitation.
+  if (sys.platform.startswith('win') and len(args) == 2
+      and input_args[1].count(' ') > 0):
+    input_args = args[:1] + args[1].split()
+
+  # Ninja uses getopt_long, which allow to intermix non-option arguments.
+  # To leave non supported parameters untouched, we do not use getopt.
+  for index, arg in enumerate(input_args[1:]):
+    if arg.startswith('-j'):
+      j_specified = True
+    if arg.startswith('-t'):
+      t_specified = True
+    if arg == '-C':
+      # + 1 to get the next argument and +1 because we trimmed off input_args[0]
+      output_dir = input_args[index + 2]
+    elif arg.startswith('-C'):
+      # Support -Cout/Default
+      output_dir = arg[2:]
+    elif arg in ('-o', '--offline'):
+      offline = True
+    elif arg == '-h':
+      print('autoninja: Use -o/--offline to temporary disable goma.',
             file=sys.stderr)
-      if sys.platform.startswith('win'):
-        # Set an exit code of 1 in the batch file.
-        print('cmd "/c exit 1"')
-      else:
-        # Set an exit code of 1 by executing 'false' in the bash script.
-        print('false')
-      sys.exit(1)
-
-# Specify ninja.exe on Windows so that ninja.bat can call autoninja and not
-# be called back.
-ninja_exe = 'ninja.exe' if sys.platform.startswith('win') else 'ninja'
-ninja_exe_path = os.path.join(SCRIPT_DIR, ninja_exe)
-
-# A large build (with or without goma) tends to hog all system resources.
-# Launching the ninja process with 'nice' priorities improves this situation.
-prefix_args = []
-if (sys.platform.startswith('linux')
-    and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'):
-  # nice -10 is process priority 10 lower than default 0
-  # ionice -c 3 is IO priority IDLE
-  prefix_args = ['nice'] + ['-10']
-
-
-# Use absolute path for ninja path,
-# or fail to execute ninja if depot_tools is not in PATH.
-args = prefix_args + [ninja_exe_path] + input_args[1:]
-
-num_cores = multiprocessing.cpu_count()
-if not j_specified and not t_specified:
-  if use_goma or use_remoteexec:
-    args.append('-j')
-    core_multiplier = int(os.environ.get('NINJA_CORE_MULTIPLIER', '40'))
-    j_value = num_cores * core_multiplier
-
-    if sys.platform.startswith('win'):
-      # On windows, j value higher than 1000 does not improve build performance.
-      j_value = min(j_value, 1000)
-    elif sys.platform == 'darwin':
-      # On Mac, j value higher than 500 causes 'Too many open files' error
-      # (crbug.com/936864).
-      j_value = min(j_value, 500)
-
-    args.append('%d' % j_value)
+      print(file=sys.stderr)
+
+  # Strip -o/--offline so ninja doesn't see them.
+  input_args = [arg for arg in input_args if arg not in ('-o', '--offline')]
+
+  use_goma = False
+  use_remoteexec = False
+
+  # Currently get reclient binary and config dirs relative to output_dir.  If
+  # they exist and using remoteexec, then automatically call bootstrap to start
+  # reproxy.  This works under the current assumption that the output
+  # directory is two levels up from chromium/src.
+  reclient_bin_dir = os.path.join(output_dir, '..', '..', 'buildtools',
+                                  'reclient')
+  reclient_cfg = os.path.join(output_dir, '..', '..', 'buildtools',
+                              'reclient_cfgs', 'reproxy.cfg')
+
+  # Attempt to auto-detect remote build acceleration. We support gn-based
+  # builds, where we look for args.gn in the build tree, and cmake-based builds
+  # where we look for rules.ninja.
+  if os.path.exists(os.path.join(output_dir, 'args.gn')):
+    with open(os.path.join(output_dir, 'args.gn')) as file_handle:
+      for line in file_handle:
+        # Either use_goma, use_remoteexec or use_rbe (in deprecation)
+        # activate build acceleration.
+        #
+        # This test can match multi-argument lines. Examples of this are:
+        # is_debug=false use_goma=true is_official_build=false
+        # use_goma=false# use_goma=true This comment is ignored
+        #
+        # Anything after a comment is not consider a valid argument.
+        line_without_comment = line.split('#')[0]
+        if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
+                     line_without_comment):
+          use_goma = True
+          continue
+        if re.search(r'(^|\s)(use_rbe|use_remoteexec)\s*=\s*true($|\s)',
+                     line_without_comment):
+          use_remoteexec = True
+          continue
   else:
-    j_value = num_cores
-    # Ninja defaults to |num_cores + 2|
-    j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2'))
-    args.append('-j')
-    args.append('%d' % j_value)
-
-# On Windows, fully quote the path so that the command processor doesn't think
-# the whole output is the command.
-# On Linux and Mac, if people put depot_tools in directories with ' ',
-# shell would misunderstand ' ' as a path separation.
-# TODO(yyanagisawa): provide proper quoting for Windows.
-# see https://cs.chromium.org/chromium/src/tools/mb/mb.py
-for i in range(len(args)):
-  if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]:
-    args[i] = '"%s"' % args[i].replace('"', '\\"')
-
-if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1':
-  args += ['-d', 'stats']
-
-# If using remoteexec and the necessary environment variables are set,
-# also start reproxy (via bootstrap) before running ninja.
-if (not offline and use_remoteexec and os.path.exists(reclient_bin_dir)
-    and os.path.exists(reclient_cfg)):
-  bootstrap = os.path.join(reclient_bin_dir, 'bootstrap')
-  setup_args = [
-    bootstrap,
-    '--cfg=' + reclient_cfg,
-    '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy')]
-
-  teardown_args = [bootstrap, '--cfg=' + reclient_cfg, '--shutdown']
-
-  cmd_sep = '\n' if sys.platform.startswith('win') else '&&'
-  args = setup_args + [cmd_sep] + args + [cmd_sep] + teardown_args
-
-if offline and not sys.platform.startswith('win'):
-  # Tell goma or reclient to do local compiles. On Windows these environment
-  # variables are set by the wrapper batch file.
-  print('RBE_remote_disabled=1 GOMA_DISABLED=1 ' + ' '.join(args))
-else:
-  print(' '.join(args))
+    for relative_path in [
+        '',  # GN keeps them in the root of output_dir
+        'CMakeFiles'
+    ]:
+      path = os.path.join(output_dir, relative_path, 'rules.ninja')
+      if os.path.exists(path):
+        with open(path) as file_handle:
+          for line in file_handle:
+            if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
+              use_goma = True
+              break
+
+  # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1"
+  # (case-insensitive) then gomacc will use the local compiler instead of doing
+  # a goma compile. This is convenient if you want to briefly disable goma. It
+  # avoids having to rebuild the world when transitioning between goma/non-goma
+  # builds. However, it is not as fast as doing a "normal" non-goma build
+  # because an extra process is created for each compile step. Checking this
+  # environment variable ensures that autoninja uses an appropriate -j value in
+  # this situation.
+  goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
+  if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
+    use_goma = False
+
+  if use_goma:
+    gomacc_file = 'gomacc.exe' if sys.platform.startswith('win') else 'gomacc'
+    goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin'))
+    gomacc_path = os.path.join(goma_dir, gomacc_file)
+    # Don't invoke gomacc if it doesn't exist.
+    if os.path.exists(gomacc_path):
+      # Check to make sure that goma is running. If not, don't start the build.
+      status = subprocess.call([gomacc_path, 'port'],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE,
+                               shell=False)
+      if status == 1:
+        print('Goma is not running. Use "goma_ctl ensure_start" to start it.',
+              file=sys.stderr)
+        if sys.platform.startswith('win'):
+          # Set an exit code of 1 in the batch file.
+          print('cmd "/c exit 1"')
+        else:
+          # Set an exit code of 1 by executing 'false' in the bash script.
+          print('false')
+        sys.exit(1)
+
+  # Specify ninja.exe on Windows so that ninja.bat can call autoninja and not
+  # be called back.
+  ninja_exe = 'ninja.exe' if sys.platform.startswith('win') else 'ninja'
+  ninja_exe_path = os.path.join(SCRIPT_DIR, ninja_exe)
+
+  # A large build (with or without goma) tends to hog all system resources.
+  # Launching the ninja process with 'nice' priorities improves this situation.
+  prefix_args = []
+  if (sys.platform.startswith('linux')
+      and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'):
+    # nice -10 is process priority 10 lower than default 0
+    # ionice -c 3 is IO priority IDLE
+    prefix_args = ['nice'] + ['-10']
+
+  # Use absolute path for ninja path,
+  # or fail to execute ninja if depot_tools is not in PATH.
+  args = prefix_args + [ninja_exe_path] + input_args[1:]
+
+  num_cores = multiprocessing.cpu_count()
+  if not j_specified and not t_specified:
+    if use_goma or use_remoteexec:
+      args.append('-j')
+      core_multiplier = int(os.environ.get('NINJA_CORE_MULTIPLIER', '40'))
+      j_value = num_cores * core_multiplier
+
+      if sys.platform.startswith('win'):
+        # On windows, j value higher than 1000 does not improve build
+        # performance.
+        j_value = min(j_value, 1000)
+      elif sys.platform == 'darwin':
+        # On Mac, j value higher than 500 causes 'Too many open files' error
+        # (crbug.com/936864).
+        j_value = min(j_value, 500)
+
+      args.append('%d' % j_value)
+    else:
+      j_value = num_cores
+      # Ninja defaults to |num_cores + 2|
+      j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2'))
+      args.append('-j')
+      args.append('%d' % j_value)
+
+  # On Windows, fully quote the path so that the command processor doesn't think
+  # the whole output is the command.
+  # On Linux and Mac, if people put depot_tools in directories with ' ',
+  # shell would misunderstand ' ' as a path separation.
+  # TODO(yyanagisawa): provide proper quoting for Windows.
+  # see https://cs.chromium.org/chromium/src/tools/mb/mb.py
+  for i in range(len(args)):
+    if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]:
+      args[i] = '"%s"' % args[i].replace('"', '\\"')
+
+  if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1':
+    args += ['-d', 'stats']
+
+  # If using remoteexec and the necessary environment variables are set,
+  # also start reproxy (via bootstrap) before running ninja.
+  if (not offline and use_remoteexec and os.path.exists(reclient_bin_dir)
+      and os.path.exists(reclient_cfg)):
+    bootstrap = os.path.join(reclient_bin_dir, 'bootstrap')
+    setup_args = [
+        bootstrap, '--cfg=' + reclient_cfg,
+        '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy')
+    ]
+
+    teardown_args = [bootstrap, '--cfg=' + reclient_cfg, '--shutdown']
+
+    cmd_sep = '\n' if sys.platform.startswith('win') else '&&'
+    args = setup_args + [cmd_sep] + args + [cmd_sep] + teardown_args
+
+  if offline and not sys.platform.startswith('win'):
+    # Tell goma or reclient to do local compiles. On Windows these environment
+    # variables are set by the wrapper batch file.
+    return 'RBE_remote_disabled=1 GOMA_DISABLED=1 ' + ' '.join(args)
+
+  return ' '.join(args)
+
+
+if __name__ == '__main__':
+  print(main(sys.argv))

tests/OWNERS

@@ -1 +1,3 @@
+per-file autoninja_test.py=brucedawson@chromium.org
+per-file autoninja_test.py=tikuta@chromium.org
 per-file ninjalog_uploader_test.py=tikuta@chromium.org

tests/autoninja_test.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import multiprocessing
+import os
+import os.path
+import sys
+import unittest
+import unittest.mock
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT_DIR)
+
+import autoninja
+
+
+class AutoninjaTest(unittest.TestCase):
+  def test_autoninja(self):
+    autoninja.main([])
+
+  def test_autoninja_goma(self):
+    with unittest.mock.patch(
+        'os.path.exists',
+        return_value=True) as mock_exists, unittest.mock.patch(
+            'autoninja.open', unittest.mock.mock_open(
+                read_data='use_goma=true')) as mock_open, unittest.mock.patch(
+                    'subprocess.call', return_value=0):
+      args = autoninja.main([]).split()
+      mock_exists.assert_called()
+      mock_open.assert_called_once()
+
+    self.assertIn('-j', args)
+    parallel_j = int(args[args.index('-j') + 1])
+    self.assertGreater(parallel_j, multiprocessing.cpu_count())
+
+
+if __name__ == '__main__':
+  unittest.main()