From 61fad561d63c63a9ac8a76f4864cce113db78f32 Mon Sep 17 00:00:00 2001 From: Fumitoshi Ukai Date: Wed, 11 Sep 2024 00:03:49 +0000 Subject: [PATCH] autoninja: check RBE project, not account Account check would become too slow. We'll check RBE project to use instead. On corp machine, our policy to use @google.com account and rbe-chrome-untrusted to build chromium/chrome. We don't allow rbe-chromium-untrusted with @chromium.org on corp machine. On non-corp machine, user couldn't use rbe-chrome-untrusted because it's @google.com only, and corp security policy doesn't allow @google.com account on non-corp machine. Bug: b/364318216 Change-Id: I0f3a19e105b050aef6a62e1b25b45b1722382a34 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5848450 Reviewed-by: Scott Lee Reviewed-by: Michael Savigny Commit-Queue: Fumitoshi Ukai Reviewed-by: Junji Watanabe Reviewed-by: Philipp Wollermann --- .gitignore | 3 - .vpython3 | 23 ----- autoninja | 2 +- autoninja.bat | 4 +- autoninja.py | 199 +++++++++++++++++----------------------- tests/autoninja_test.py | 43 ++------- 6 files changed, 93 insertions(+), 181 deletions(-) diff --git a/.gitignore b/.gitignore index c56bc41566..ff5b23ff54 100644 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,3 @@ testing_support/google_appengine # Ignore the file that logs Python 2 scripts run during presubmits. /python2_usage.txt - -# Ignore the internal data used by autoninja. -/.autoninja* diff --git a/.vpython3 b/.vpython3 index e27508d1f9..484320edd8 100644 --- a/.vpython3 +++ b/.vpython3 @@ -75,29 +75,6 @@ wheel: < version: "version:2021.5.30" > -# Used by: -# autoninja.py -wheel: < - name: "infra/python/wheels/google-auth-py3" - version: "version:2.16.2" -> -wheel: < - name: "infra/python/wheels/cachetools-py3" - version: "version:4.2.2" -> -wheel: < - name: "infra/python/wheels/pyasn1_modules-py2_py3" - version: "version:0.2.8" -> -wheel: < - name: "infra/python/wheels/rsa-py3" - version: "version:4.7.2" -> -wheel: < - name: "infra/python/wheels/pyasn1-py2_py3" - version: "version:0.4.8" -> - # Used by: # tests/autoninja_test.py wheel: < diff --git a/autoninja b/autoninja index 2a33a2fa0e..7fa431b0ae 100755 --- a/autoninja +++ b/autoninja @@ -16,7 +16,7 @@ fi # Execute whatever is printed by autoninja.py. # Also print it to reassure that the right settings are being used. -vpython3 "$(dirname -- "$0")/autoninja.py" "$@" +python3 "$(dirname -- "$0")/autoninja.py" "$@" retval=$? if [ "$retval" == "0" ] && [ "$NINJA_SUMMARIZE_BUILD" == "1" ]; then diff --git a/autoninja.bat b/autoninja.bat index ae8178f619..4526bb8091 100755 --- a/autoninja.bat +++ b/autoninja.bat @@ -9,7 +9,7 @@ set scriptdir=%~dp0 if "%*" == "/?" ( rem Handle "autoninja /?" which will otherwise give help on the "call" command - @call python3.bat %~dp0\ninja.py --help + @call %scriptdir%python-bin\python3.bat %~dp0\ninja.py --help exit /b ) @@ -20,7 +20,7 @@ if "%*" == "/?" ( if "%NINJA_SUMMARIZE_BUILD%" == "1" set "NINJA_STATUS=[%%r processes, %%f/%%t @ %%o/s : %%es ] " :: Execute autoninja.py and pass all arguments to it. -@call %scriptdir%\vpython3.bat %scriptdir%autoninja.py "%%*" +@call %scriptdir%python-bin\python3.bat %scriptdir%autoninja.py "%%*" @if errorlevel 1 goto buildfailure :: Use call to invoke python script here, because we use python via python3.bat. diff --git a/autoninja.py b/autoninja.py index 0d1c4cf05f..6282e5f4bf 100755 --- a/autoninja.py +++ b/autoninja.py @@ -1,4 +1,4 @@ -#!/usr/bin/env vpython3 +#!/usr/bin/env python3 # Copyright (c) 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -16,12 +16,10 @@ settings. import uuid import logging -import json import multiprocessing import os import platform import re -import shelve import shlex import shutil import subprocess @@ -29,10 +27,8 @@ import sys import time import warnings -import google.auth -from google.auth.transport.requests import AuthorizedSession - import build_telemetry +import gclient_paths import gclient_utils import gn_helper import ninja @@ -57,114 +53,46 @@ _UNSAFE_FOR_CMD = set("^<>&|()%") _ALL_META_CHARS = _UNSAFE_FOR_CMD.union(set('"')) -def _adc_account(): - """Returns account used to authenticate with GCP application default credentials.""" - - try: - # Suppress warnings from google.auth.default. - # https://github.com/googleapis/google-auth-library-python/issues/271 - warnings.filterwarnings( - "ignore", - "Your application has authenticated using end user credentials from" - " Google Cloud SDK without a quota project.", - ) - credentials, _ = google.auth.default( - scopes=["https://www.googleapis.com/auth/userinfo.email"]) - except google.auth.exceptions.DefaultCredentialsError: - # Application Default Crendetials is not configured. - return None - finally: - warnings.resetwarnings() - - with AuthorizedSession(credentials) as session: - try: - response = session.get( - "https://www.googleapis.com/oauth2/v1/userinfo") - except Exception: - # Ignore exception. - return None - - return response.json().get("email") - - -def _gcloud_auth_account(): - """Returns active account authenticated with `gcloud auth login`.""" - if shutil.which("gcloud") is None: - return None - - accounts = json.loads( - subprocess.check_output("gcloud auth list --format=json", - shell=True, - text=True)) - for account in accounts: - if account["status"] == "ACTIVE": - return account["account"] - return None - - -def _luci_auth_account(): - """Returns active account authenticated with `luci-auth login -scopes-context`.""" - if shutil.which("luci-auth") is None: - return None - - # First line returned should be "Logged in as account@domain.com." - # Extract the account@domain.com from that line. - try: - info = subprocess.check_output("luci-auth info -scopes-context", - shell=True, - stderr=subprocess.STDOUT, - text=True).split('\n')[0] - if info.startswith("Logged in as "): - return info[len("Logged in as "):-1] - except subprocess.CalledProcessError: - return None - return None - - def _is_google_corp_machine(): """This assumes that corp machine has gcert binary in known location.""" return shutil.which("gcert") is not None -def _is_google_corp_machine_using_external_account(): - if os.environ.get("AUTONINJA_SKIP_EXTERNAL_ACCOUNT_CHECK") == "1": - print( - "WARNING: AUTONINJA_SKIP_EXTERNAL_ACCOUNT_CHECK env var is set.\n" - "This is only for some infra, do not set this in personal" - " development machine.", - file=sys.stderr) - return False +def _reclient_rbe_project(): + """Returns RBE project used by reclient.""" + instance = os.environ.get('RBE_instance') + if instance: + m = re.match(instance, 'projects/([^/]*)/instances/.*') + if m: + return m[1] + reproxy_cfg_path = reclient_helper.find_reclient_cfg() + if not reproxy_cfg_path: + return "" + with open(reproxy_cfg_path) as f: + for line in f: + m = re.match('instance\s*=\s*projects/([^/]*)/instances/.*', line) + if m: + return m[1] + return "" - if not _is_google_corp_machine(): - return False - with shelve.open(os.path.join(_SCRIPT_DIR, ".autoninja")) as db: - last_false = db.get("last_false") - now = time.time() - if last_false is not None and now < last_false + 12 * 60 * 60: - # Do not check account if it is checked in last 12 hours. - return False - - account = _adc_account() - if account and not account.endswith("@google.com"): - return True - - account = _luci_auth_account() - if account and not account.endswith("@google.com"): - return True - - account = _gcloud_auth_account() - if not account: - db["last_false"] = now - return False - - # Handle service account and google account as internal account. - if not (account.endswith("@google.com") - or account.endswith("gserviceaccount.com")): - return True - - db["last_false"] = now - return False +def _siso_rbe_project(): + """Returns RBE project used by siso.""" + siso_project = os.environ.get('SISO_PROJECT') + if siso_project: + return siso_project + root_dir = gclient_paths.GetPrimarySolutionPath() + if not root_dir: + return "" + sisoenv_path = os.path.join(root_dir, 'build/config/siso/.sisoenv') + if not os.path.exists(sisoenv_path): + return "" + with open(sisoenv_path) as f: + for line in f: + m = re.match('SISO_PROJECT=\s*(\S*)\s*', line) + if m: + return m[1] + return "" def _quote_for_cmd(arg): @@ -200,6 +128,7 @@ def _main_inner(input_args, build_id, should_collect_logs=False): offline = False output_dir = "." summarize_build = os.environ.get("NINJA_SUMMARIZE_BUILD") == "1" + project = None # Ninja uses getopt_long, which allow to intermix non-option arguments. # To leave non supported parameters untouched, we do not use getopt. @@ -217,6 +146,12 @@ def _main_inner(input_args, build_id, should_collect_logs=False): output_dir = arg[2:] elif arg in ("-o", "--offline"): offline = True + elif arg in ("--project", "-project"): + project = input_args[index + 2] + elif arg.startswith("--project="): + project = arg[len("--project="):] + elif arg.startswith("-project="): + project = arg[len("-project="):] elif arg in ("-h", "--help"): print( "autoninja: Use -o/--offline to temporary disable remote execution.", @@ -262,16 +197,46 @@ def _main_inner(input_args, build_id, should_collect_logs=False): use_reclient = use_remoteexec if use_remoteexec: - if _is_google_corp_machine_using_external_account(): - print( - "You can't use a non-@google.com account (%s and/or %s) on" - " a corp machine.\n" - "Please login via `gcloud auth login --update-adc` with" - " your @google.com account instead.\n" % - (_adc_account(), _gcloud_auth_account()), - file=sys.stderr, - ) - return 1 + if use_reclient: + project = _reclient_rbe_project() + if not project: + print( + "Can't detect RBE project to use.\n" + "Did you setup properly?\n", + file=sys.stderr, + ) + return 1 + elif use_siso and project is None: + # siso runs locally if empty project is given + # even if use_remoteexec=true is set. + project = _siso_rbe_project() + + if _is_google_corp_machine(): + # user may login on non-@google.com account on corp, + # but need to use @google.com and rbe-chrome-untrusted + # on corp machine. + if project == 'rbe-chromium-untrusted': + print( + "You can't use rbe-chromium-untrusted on corp " + "machine.\n" + "Please use rbe-chrome-untrusted and @google.com " + "account instead to build chromium.\n", + file=sys.stderr, + ) + return 1 + else: + # only @google.com is allowed to use rbe-chrome-untrusted + # and use @google.com on non-corp machine is not allowed + # by corp security policy. + if project == 'rbe-chrome-untrusted': + print( + "You can't use rbe-chrome-untrusted on non-corp " + "machine.\n" + "Plase use rbe-chromium-untrusted and non-@google.com " + "account instead to build chromium.", + file=sys.stderr, + ) + return 1 if gclient_utils.IsEnvCog(): if not use_remoteexec or use_reclient or not use_siso: diff --git a/tests/autoninja_test.py b/tests/autoninja_test.py index e5b4a8e956..d7cecd99dd 100755 --- a/tests/autoninja_test.py +++ b/tests/autoninja_test.py @@ -85,6 +85,10 @@ class AutoninjaTest(trial_dir.TestCase): write(os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'), 'RBE_v=2') write(os.path.join('buildtools', 'reclient', 'version.txt'), '0.0') + write( + os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'), + 'instance=projects/rbe-chromium-untrusted-test/' + 'instances/default_instance') autoninja.main(['autoninja.py', '-C', out_dir]) run_ninja.assert_called_once() args = run_ninja.call_args.args[0] @@ -104,6 +108,8 @@ class AutoninjaTest(trial_dir.TestCase): with mock.patch('siso.main', return_value=0) as siso_main: out_dir = os.path.join('out', 'dir') write(os.path.join(out_dir, 'args.gn'), 'use_siso=true') + write(os.path.join('build', 'config', 'siso', '.sisoenv'), + 'SISO_PROJECT=rbe-chromium-untrusted-test') autoninja.main(['autoninja.py', '-C', out_dir]) siso_main.assert_called_once() args = siso_main.call_args.args[0] @@ -129,6 +135,8 @@ class AutoninjaTest(trial_dir.TestCase): 'use_siso=true\nuse_remoteexec=true') write( os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'), + 'instance=projects/rbe-chromium-untrusted-test/' + 'instances/default_instance\n' 'RBE_v=2') write(os.path.join('buildtools', 'reclient', 'version.txt'), '0.0') @@ -143,41 +151,6 @@ class AutoninjaTest(trial_dir.TestCase): ['siso', 'ninja', '-project=', '-reapi_instance=', '-C', out_dir]) self.assertEqual(reclient_helper_calls[0][1], 'autosiso') - @parameterized.expand([ - ("non corp machine", False, None, None, None, False), - ("non corp adc account", True, "foo@chromium.org", None, None, True), - ("corp adc account", True, "foo@google.com", None, None, False), - ("non corp gcloud auth account", True, None, "foo@chromium.org", None, - True), - ("corp gcloud auth account", True, None, "foo@google.com", None, False), - ("non corp luci auth account", True, None, None, "foo@chromium.org", - True), - ("corp luci auth account", True, None, None, "foo@google.com", False), - ]) - def test_is_corp_machine_using_external_account(self, _, is_corp, - adc_account, - gcloud_auth_account, - luci_auth_account, - expected): - for shelve_file in glob.glob( - os.path.join(autoninja._SCRIPT_DIR, ".autoninja*")): - # Clear cache. - os.remove(shelve_file) - - with mock.patch('autoninja._is_google_corp_machine', - return_value=is_corp), mock.patch( - 'autoninja._adc_account', - return_value=adc_account), mock.patch( - 'autoninja._gcloud_auth_account', - return_value=gcloud_auth_account), mock.patch( - 'autoninja._luci_auth_account', - return_value=luci_auth_account): - self.assertEqual( - bool( - # pylint: disable=line-too-long - autoninja._is_google_corp_machine_using_external_account()), - expected) - @mock.patch('sys.platform', 'win32') def test_print_cmd_windows(self): args = [