autoninja: check RBE project, not account

Account check would become too slow.
We'll check RBE project to use instead.

On corp machine, our policy to use @google.com account
and rbe-chrome-untrusted to build chromium/chrome.
We don't allow rbe-chromium-untrusted with @chromium.org
on corp machine.

On non-corp machine, user couldn't use rbe-chrome-untrusted
because it's @google.com only, and corp security policy
doesn't allow @google.com account on non-corp machine.

Bug: b/364318216
Change-Id: I0f3a19e105b050aef6a62e1b25b45b1722382a34
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5848450
Reviewed-by: Scott Lee <ddoman@chromium.org>
Reviewed-by: Michael Savigny <msavigny@google.com>
Commit-Queue: Fumitoshi Ukai <ukai@google.com>
Reviewed-by: Junji Watanabe <jwata@google.com>
Reviewed-by: Philipp Wollermann <philwo@google.com>

.gitignore

@@ -96,6 +96,3 @@ testing_support/google_appengine
 
 # Ignore the file that logs Python 2 scripts run during presubmits.
 /python2_usage.txt
-
-# Ignore the internal data used by autoninja.
-/.autoninja*

.vpython3

@@ -75,29 +75,6 @@ wheel: <
   version: "version:2021.5.30"
 >
 
-# Used by:
-#   autoninja.py
-wheel: <
-  name: "infra/python/wheels/google-auth-py3"
-  version: "version:2.16.2"
->
-wheel: <
-  name: "infra/python/wheels/cachetools-py3"
-  version: "version:4.2.2"
->
-wheel: <
-  name: "infra/python/wheels/pyasn1_modules-py2_py3"
-  version: "version:0.2.8"
->
-wheel: <
-  name: "infra/python/wheels/rsa-py3"
-  version: "version:4.7.2"
->
-wheel: <
-  name: "infra/python/wheels/pyasn1-py2_py3"
-  version: "version:0.4.8"
->
-
 # Used by:
 #   tests/autoninja_test.py
 wheel: <

autoninja

@@ -16,7 +16,7 @@ fi
 
 # Execute whatever is printed by autoninja.py.
 # Also print it to reassure that the right settings are being used.
-vpython3 "$(dirname -- "$0")/autoninja.py" "$@"
+python3 "$(dirname -- "$0")/autoninja.py" "$@"
 retval=$?
 
 if [ "$retval" == "0" ] && [ "$NINJA_SUMMARIZE_BUILD" == "1" ]; then

autoninja.bat

@@ -9,7 +9,7 @@ set scriptdir=%~dp0
 
 if "%*" == "/?" (
   rem Handle "autoninja /?" which will otherwise give help on the "call" command
-  @call python3.bat %~dp0\ninja.py --help
+  @call %scriptdir%python-bin\python3.bat %~dp0\ninja.py --help
   exit /b
 )
 
@@ -20,7 +20,7 @@ if "%*" == "/?" (
 if "%NINJA_SUMMARIZE_BUILD%" == "1" set "NINJA_STATUS=[%%r processes, %%f/%%t @ %%o/s : %%es ] "
 
 :: Execute autoninja.py and pass all arguments to it.
-@call %scriptdir%\vpython3.bat %scriptdir%autoninja.py "%%*"
+@call %scriptdir%python-bin\python3.bat %scriptdir%autoninja.py "%%*"
 @if errorlevel 1 goto buildfailure
 
 :: Use call to invoke python script here, because we use python via python3.bat.

autoninja.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env vpython3
+#!/usr/bin/env python3
 # Copyright (c) 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -16,12 +16,10 @@ settings.
 
 import uuid
 import logging
-import json
 import multiprocessing
 import os
 import platform
 import re
-import shelve
 import shlex
 import shutil
 import subprocess
@@ -29,10 +27,8 @@ import sys
 import time
 import warnings
 
-import google.auth
-from google.auth.transport.requests import AuthorizedSession
-
 import build_telemetry
+import gclient_paths
 import gclient_utils
 import gn_helper
 import ninja
@@ -57,114 +53,46 @@ _UNSAFE_FOR_CMD = set("^<>&|()%")
 _ALL_META_CHARS = _UNSAFE_FOR_CMD.union(set('"'))
 
 
-def _adc_account():
-    """Returns account used to authenticate with GCP application default credentials."""
-
-    try:
-        # Suppress warnings from google.auth.default.
-        # https://github.com/googleapis/google-auth-library-python/issues/271
-        warnings.filterwarnings(
-            "ignore",
-            "Your application has authenticated using end user credentials from"
-            " Google Cloud SDK without a quota project.",
-        )
-        credentials, _ = google.auth.default(
-            scopes=["https://www.googleapis.com/auth/userinfo.email"])
-    except google.auth.exceptions.DefaultCredentialsError:
-        # Application Default Crendetials is not configured.
-        return None
-    finally:
-        warnings.resetwarnings()
-
-    with AuthorizedSession(credentials) as session:
-        try:
-            response = session.get(
-                "https://www.googleapis.com/oauth2/v1/userinfo")
-        except Exception:
-            # Ignore exception.
-            return None
-
-    return response.json().get("email")
-
-
-def _gcloud_auth_account():
-    """Returns active account authenticated with `gcloud auth login`."""
-    if shutil.which("gcloud") is None:
-        return None
-
-    accounts = json.loads(
-        subprocess.check_output("gcloud auth list --format=json",
-                                shell=True,
-                                text=True))
-    for account in accounts:
-        if account["status"] == "ACTIVE":
-            return account["account"]
-    return None
-
-
-def _luci_auth_account():
-    """Returns active account authenticated with `luci-auth login -scopes-context`."""
-    if shutil.which("luci-auth") is None:
-        return None
-
-    # First line returned should be "Logged in as account@domain.com."
-    # Extract the account@domain.com from that line.
-    try:
-        info = subprocess.check_output("luci-auth info -scopes-context",
-                                       shell=True,
-                                       stderr=subprocess.STDOUT,
-                                       text=True).split('\n')[0]
-        if info.startswith("Logged in as "):
-            return info[len("Logged in as "):-1]
-    except subprocess.CalledProcessError:
-        return None
-    return None
-
-
 def _is_google_corp_machine():
     """This assumes that corp machine has gcert binary in known location."""
     return shutil.which("gcert") is not None
 
 
-def _is_google_corp_machine_using_external_account():
-    if os.environ.get("AUTONINJA_SKIP_EXTERNAL_ACCOUNT_CHECK") == "1":
-        print(
-            "WARNING: AUTONINJA_SKIP_EXTERNAL_ACCOUNT_CHECK env var is set.\n"
-            "This is only for some infra, do not set this in personal"
-            " development machine.",
-            file=sys.stderr)
-        return False
-
-    if not _is_google_corp_machine():
-        return False
-
-    with shelve.open(os.path.join(_SCRIPT_DIR, ".autoninja")) as db:
-        last_false = db.get("last_false")
-        now = time.time()
-        if last_false is not None and now < last_false + 12 * 60 * 60:
-            # Do not check account if it is checked in last 12 hours.
-            return False
-
-        account = _adc_account()
-        if account and not account.endswith("@google.com"):
-            return True
-
-        account = _luci_auth_account()
-        if account and not account.endswith("@google.com"):
-            return True
-
-        account = _gcloud_auth_account()
-        if not account:
-            db["last_false"] = now
-            return False
-
-        # Handle service account and google account as internal account.
-        if not (account.endswith("@google.com")
-                or account.endswith("gserviceaccount.com")):
-            return True
-
-        db["last_false"] = now
-        return False
+def _reclient_rbe_project():
+    """Returns RBE project used by reclient."""
+    instance = os.environ.get('RBE_instance')
+    if instance:
+        m = re.match(instance, 'projects/([^/]*)/instances/.*')
+        if m:
+            return m[1]
+    reproxy_cfg_path = reclient_helper.find_reclient_cfg()
+    if not reproxy_cfg_path:
+        return ""
+    with open(reproxy_cfg_path) as f:
+        for line in f:
+            m = re.match('instance\s*=\s*projects/([^/]*)/instances/.*', line)
+            if m:
+                return m[1]
+    return ""
+
+
+def _siso_rbe_project():
+    """Returns RBE project used by siso."""
+    siso_project = os.environ.get('SISO_PROJECT')
+    if siso_project:
+        return siso_project
+    root_dir = gclient_paths.GetPrimarySolutionPath()
+    if not root_dir:
+        return ""
+    sisoenv_path = os.path.join(root_dir, 'build/config/siso/.sisoenv')
+    if not os.path.exists(sisoenv_path):
+        return ""
+    with open(sisoenv_path) as f:
+        for line in f:
+            m = re.match('SISO_PROJECT=\s*(\S*)\s*', line)
+            if m:
+                return m[1]
+    return ""
 
 
 def _quote_for_cmd(arg):
@@ -200,6 +128,7 @@ def _main_inner(input_args, build_id, should_collect_logs=False):
     offline = False
     output_dir = "."
     summarize_build = os.environ.get("NINJA_SUMMARIZE_BUILD") == "1"
+    project = None
 
     # Ninja uses getopt_long, which allow to intermix non-option arguments.
     # To leave non supported parameters untouched, we do not use getopt.
@@ -217,6 +146,12 @@ def _main_inner(input_args, build_id, should_collect_logs=False):
             output_dir = arg[2:]
         elif arg in ("-o", "--offline"):
             offline = True
+        elif arg in ("--project", "-project"):
+            project = input_args[index + 2]
+        elif arg.startswith("--project="):
+            project = arg[len("--project="):]
+        elif arg.startswith("-project="):
+            project = arg[len("-project="):]
         elif arg in ("-h", "--help"):
             print(
                 "autoninja: Use -o/--offline to temporary disable remote execution.",
@@ -262,16 +197,46 @@ def _main_inner(input_args, build_id, should_collect_logs=False):
             use_reclient = use_remoteexec
 
         if use_remoteexec:
-            if _is_google_corp_machine_using_external_account():
-                print(
-                    "You can't use a non-@google.com account (%s and/or %s) on"
-                    " a corp machine.\n"
-                    "Please login via `gcloud auth login --update-adc` with"
-                    " your @google.com account instead.\n" %
-                    (_adc_account(), _gcloud_auth_account()),
-                    file=sys.stderr,
-                )
-                return 1
+            if use_reclient:
+                project = _reclient_rbe_project()
+                if not project:
+                    print(
+                        "Can't detect RBE project to use.\n"
+                        "Did you setup properly?\n",
+                        file=sys.stderr,
+                    )
+                    return 1
+            elif use_siso and project is None:
+                # siso runs locally if empty project is given
+                # even if use_remoteexec=true is set.
+                project = _siso_rbe_project()
+
+            if _is_google_corp_machine():
+                # user may login on non-@google.com account on corp,
+                # but need to use @google.com and rbe-chrome-untrusted
+                # on corp machine.
+                if project == 'rbe-chromium-untrusted':
+                    print(
+                        "You can't use rbe-chromium-untrusted on corp "
+                        "machine.\n"
+                        "Please use rbe-chrome-untrusted and @google.com "
+                        "account instead to build chromium.\n",
+                        file=sys.stderr,
+                    )
+                    return 1
+            else:
+                # only @google.com is allowed to use rbe-chrome-untrusted
+                # and use @google.com on non-corp machine is not allowed
+                # by corp security policy.
+                if project == 'rbe-chrome-untrusted':
+                    print(
+                        "You can't use rbe-chrome-untrusted on non-corp "
+                        "machine.\n"
+                        "Plase use rbe-chromium-untrusted and non-@google.com "
+                        "account instead to build chromium.",
+                        file=sys.stderr,
+                    )
+                    return 1
 
         if gclient_utils.IsEnvCog():
             if not use_remoteexec or use_reclient or not use_siso:

tests/autoninja_test.py

@@ -85,6 +85,10 @@ class AutoninjaTest(trial_dir.TestCase):
             write(os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'),
                   'RBE_v=2')
             write(os.path.join('buildtools', 'reclient', 'version.txt'), '0.0')
+            write(
+                os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'),
+                'instance=projects/rbe-chromium-untrusted-test/'
+                'instances/default_instance')
             autoninja.main(['autoninja.py', '-C', out_dir])
             run_ninja.assert_called_once()
             args = run_ninja.call_args.args[0]
@@ -104,6 +108,8 @@ class AutoninjaTest(trial_dir.TestCase):
         with mock.patch('siso.main', return_value=0) as siso_main:
             out_dir = os.path.join('out', 'dir')
             write(os.path.join(out_dir, 'args.gn'), 'use_siso=true')
+            write(os.path.join('build', 'config', 'siso', '.sisoenv'),
+                  'SISO_PROJECT=rbe-chromium-untrusted-test')
             autoninja.main(['autoninja.py', '-C', out_dir])
             siso_main.assert_called_once()
             args = siso_main.call_args.args[0]
@@ -129,6 +135,8 @@ class AutoninjaTest(trial_dir.TestCase):
                       'use_siso=true\nuse_remoteexec=true')
                 write(
                     os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'),
+                    'instance=projects/rbe-chromium-untrusted-test/'
+                    'instances/default_instance\n'
                     'RBE_v=2')
                 write(os.path.join('buildtools', 'reclient', 'version.txt'),
                       '0.0')
@@ -143,41 +151,6 @@ class AutoninjaTest(trial_dir.TestCase):
             ['siso', 'ninja', '-project=', '-reapi_instance=', '-C', out_dir])
         self.assertEqual(reclient_helper_calls[0][1], 'autosiso')
 
-    @parameterized.expand([
-        ("non corp machine", False, None, None, None, False),
-        ("non corp adc account", True, "foo@chromium.org", None, None, True),
-        ("corp adc account", True, "foo@google.com", None, None, False),
-        ("non corp gcloud auth account", True, None, "foo@chromium.org", None,
-         True),
-        ("corp gcloud auth account", True, None, "foo@google.com", None, False),
-        ("non corp luci auth account", True, None, None, "foo@chromium.org",
-         True),
-        ("corp luci auth account", True, None, None, "foo@google.com", False),
-    ])
-    def test_is_corp_machine_using_external_account(self, _, is_corp,
-                                                    adc_account,
-                                                    gcloud_auth_account,
-                                                    luci_auth_account,
-                                                    expected):
-        for shelve_file in glob.glob(
-                os.path.join(autoninja._SCRIPT_DIR, ".autoninja*")):
-            # Clear cache.
-            os.remove(shelve_file)
-
-        with mock.patch('autoninja._is_google_corp_machine',
-                        return_value=is_corp), mock.patch(
-                            'autoninja._adc_account',
-                            return_value=adc_account), mock.patch(
-                                'autoninja._gcloud_auth_account',
-                                return_value=gcloud_auth_account), mock.patch(
-                                    'autoninja._luci_auth_account',
-                                    return_value=luci_auth_account):
-            self.assertEqual(
-                bool(
-                    # pylint: disable=line-too-long
-                    autoninja._is_google_corp_machine_using_external_account()),
-                expected)
-
     @mock.patch('sys.platform', 'win32')
     def test_print_cmd_windows(self):
         args = [