mirror of
https://chromium.googlesource.com/chromium/tools/depot_tools.git
synced 2026-01-11 10:41:31 +00:00
autoninja: check RBE project, not account
Account check would become too slow. We'll check RBE project to use instead. On corp machine, our policy to use @google.com account and rbe-chrome-untrusted to build chromium/chrome. We don't allow rbe-chromium-untrusted with @chromium.org on corp machine. On non-corp machine, user couldn't use rbe-chrome-untrusted because it's @google.com only, and corp security policy doesn't allow @google.com account on non-corp machine. Bug: b/364318216 Change-Id: I0f3a19e105b050aef6a62e1b25b45b1722382a34 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5848450 Reviewed-by: Scott Lee <ddoman@chromium.org> Reviewed-by: Michael Savigny <msavigny@google.com> Commit-Queue: Fumitoshi Ukai <ukai@google.com> Reviewed-by: Junji Watanabe <jwata@google.com> Reviewed-by: Philipp Wollermann <philwo@google.com>
This commit is contained in:
Fumitoshi Ukai
3
.gitignore
vendored
3
.gitignore
vendored
@@ -96,6 +96,3 @@ testing_support/google_appengine
|
||||
|
||||
# Ignore the file that logs Python 2 scripts run during presubmits.
|
||||
/python2_usage.txt
|
||||
|
||||
# Ignore the internal data used by autoninja.
|
||||
/.autoninja*
|
||||
|
||||
23
.vpython3
23
.vpython3
@@ -75,29 +75,6 @@ wheel: <
|
||||
version: "version:2021.5.30"
|
||||
>
|
||||
|
||||
# Used by:
|
||||
# autoninja.py
|
||||
wheel: <
|
||||
name: "infra/python/wheels/google-auth-py3"
|
||||
version: "version:2.16.2"
|
||||
>
|
||||
wheel: <
|
||||
name: "infra/python/wheels/cachetools-py3"
|
||||
version: "version:4.2.2"
|
||||
>
|
||||
wheel: <
|
||||
name: "infra/python/wheels/pyasn1_modules-py2_py3"
|
||||
version: "version:0.2.8"
|
||||
>
|
||||
wheel: <
|
||||
name: "infra/python/wheels/rsa-py3"
|
||||
version: "version:4.7.2"
|
||||
>
|
||||
wheel: <
|
||||
name: "infra/python/wheels/pyasn1-py2_py3"
|
||||
version: "version:0.4.8"
|
||||
>
|
||||
|
||||
# Used by:
|
||||
# tests/autoninja_test.py
|
||||
wheel: <
|
||||
|
||||
@@ -16,7 +16,7 @@ fi
|
||||
|
||||
# Execute whatever is printed by autoninja.py.
|
||||
# Also print it to reassure that the right settings are being used.
|
||||
vpython3 "$(dirname -- "$0")/autoninja.py" "$@"
|
||||
python3 "$(dirname -- "$0")/autoninja.py" "$@"
|
||||
retval=$?
|
||||
|
||||
if [ "$retval" == "0" ] && [ "$NINJA_SUMMARIZE_BUILD" == "1" ]; then
|
||||
|
||||
@@ -9,7 +9,7 @@ set scriptdir=%~dp0
|
||||
|
||||
if "%*" == "/?" (
|
||||
rem Handle "autoninja /?" which will otherwise give help on the "call" command
|
||||
@call python3.bat %~dp0\ninja.py --help
|
||||
@call %scriptdir%python-bin\python3.bat %~dp0\ninja.py --help
|
||||
exit /b
|
||||
)
|
||||
|
||||
@@ -20,7 +20,7 @@ if "%*" == "/?" (
|
||||
if "%NINJA_SUMMARIZE_BUILD%" == "1" set "NINJA_STATUS=[%%r processes, %%f/%%t @ %%o/s : %%es ] "
|
||||
|
||||
:: Execute autoninja.py and pass all arguments to it.
|
||||
@call %scriptdir%\vpython3.bat %scriptdir%autoninja.py "%%*"
|
||||
@call %scriptdir%python-bin\python3.bat %scriptdir%autoninja.py "%%*"
|
||||
@if errorlevel 1 goto buildfailure
|
||||
|
||||
:: Use call to invoke python script here, because we use python via python3.bat.
|
||||
|
||||
191
autoninja.py
191
autoninja.py
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env vpython3
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2017 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
@@ -16,12 +16,10 @@ settings.
|
||||
|
||||
import uuid
|
||||
import logging
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shelve
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -29,10 +27,8 @@ import sys
|
||||
import time
|
||||
import warnings
|
||||
|
||||
import google.auth
|
||||
from google.auth.transport.requests import AuthorizedSession
|
||||
|
||||
import build_telemetry
|
||||
import gclient_paths
|
||||
import gclient_utils
|
||||
import gn_helper
|
||||
import ninja
|
||||
@@ -57,114 +53,46 @@ _UNSAFE_FOR_CMD = set("^<>&|()%")
|
||||
_ALL_META_CHARS = _UNSAFE_FOR_CMD.union(set('"'))
|
||||
|
||||
|
||||
def _adc_account():
|
||||
"""Returns account used to authenticate with GCP application default credentials."""
|
||||
|
||||
try:
|
||||
# Suppress warnings from google.auth.default.
|
||||
# https://github.com/googleapis/google-auth-library-python/issues/271
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
"Your application has authenticated using end user credentials from"
|
||||
" Google Cloud SDK without a quota project.",
|
||||
)
|
||||
credentials, _ = google.auth.default(
|
||||
scopes=["https://www.googleapis.com/auth/userinfo.email"])
|
||||
except google.auth.exceptions.DefaultCredentialsError:
|
||||
# Application Default Crendetials is not configured.
|
||||
return None
|
||||
finally:
|
||||
warnings.resetwarnings()
|
||||
|
||||
with AuthorizedSession(credentials) as session:
|
||||
try:
|
||||
response = session.get(
|
||||
"https://www.googleapis.com/oauth2/v1/userinfo")
|
||||
except Exception:
|
||||
# Ignore exception.
|
||||
return None
|
||||
|
||||
return response.json().get("email")
|
||||
|
||||
|
||||
def _gcloud_auth_account():
|
||||
"""Returns active account authenticated with `gcloud auth login`."""
|
||||
if shutil.which("gcloud") is None:
|
||||
return None
|
||||
|
||||
accounts = json.loads(
|
||||
subprocess.check_output("gcloud auth list --format=json",
|
||||
shell=True,
|
||||
text=True))
|
||||
for account in accounts:
|
||||
if account["status"] == "ACTIVE":
|
||||
return account["account"]
|
||||
return None
|
||||
|
||||
|
||||
def _luci_auth_account():
|
||||
"""Returns active account authenticated with `luci-auth login -scopes-context`."""
|
||||
if shutil.which("luci-auth") is None:
|
||||
return None
|
||||
|
||||
# First line returned should be "Logged in as account@domain.com."
|
||||
# Extract the account@domain.com from that line.
|
||||
try:
|
||||
info = subprocess.check_output("luci-auth info -scopes-context",
|
||||
shell=True,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True).split('\n')[0]
|
||||
if info.startswith("Logged in as "):
|
||||
return info[len("Logged in as "):-1]
|
||||
except subprocess.CalledProcessError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _is_google_corp_machine():
|
||||
"""This assumes that corp machine has gcert binary in known location."""
|
||||
return shutil.which("gcert") is not None
|
||||
|
||||
|
||||
def _is_google_corp_machine_using_external_account():
|
||||
if os.environ.get("AUTONINJA_SKIP_EXTERNAL_ACCOUNT_CHECK") == "1":
|
||||
print(
|
||||
"WARNING: AUTONINJA_SKIP_EXTERNAL_ACCOUNT_CHECK env var is set.\n"
|
||||
"This is only for some infra, do not set this in personal"
|
||||
" development machine.",
|
||||
file=sys.stderr)
|
||||
return False
|
||||
def _reclient_rbe_project():
|
||||
"""Returns RBE project used by reclient."""
|
||||
instance = os.environ.get('RBE_instance')
|
||||
if instance:
|
||||
m = re.match(instance, 'projects/([^/]*)/instances/.*')
|
||||
if m:
|
||||
return m[1]
|
||||
reproxy_cfg_path = reclient_helper.find_reclient_cfg()
|
||||
if not reproxy_cfg_path:
|
||||
return ""
|
||||
with open(reproxy_cfg_path) as f:
|
||||
for line in f:
|
||||
m = re.match('instance\s*=\s*projects/([^/]*)/instances/.*', line)
|
||||
if m:
|
||||
return m[1]
|
||||
return ""
|
||||
|
||||
if not _is_google_corp_machine():
|
||||
return False
|
||||
|
||||
with shelve.open(os.path.join(_SCRIPT_DIR, ".autoninja")) as db:
|
||||
last_false = db.get("last_false")
|
||||
now = time.time()
|
||||
if last_false is not None and now < last_false + 12 * 60 * 60:
|
||||
# Do not check account if it is checked in last 12 hours.
|
||||
return False
|
||||
|
||||
account = _adc_account()
|
||||
if account and not account.endswith("@google.com"):
|
||||
return True
|
||||
|
||||
account = _luci_auth_account()
|
||||
if account and not account.endswith("@google.com"):
|
||||
return True
|
||||
|
||||
account = _gcloud_auth_account()
|
||||
if not account:
|
||||
db["last_false"] = now
|
||||
return False
|
||||
|
||||
# Handle service account and google account as internal account.
|
||||
if not (account.endswith("@google.com")
|
||||
or account.endswith("gserviceaccount.com")):
|
||||
return True
|
||||
|
||||
db["last_false"] = now
|
||||
return False
|
||||
def _siso_rbe_project():
|
||||
"""Returns RBE project used by siso."""
|
||||
siso_project = os.environ.get('SISO_PROJECT')
|
||||
if siso_project:
|
||||
return siso_project
|
||||
root_dir = gclient_paths.GetPrimarySolutionPath()
|
||||
if not root_dir:
|
||||
return ""
|
||||
sisoenv_path = os.path.join(root_dir, 'build/config/siso/.sisoenv')
|
||||
if not os.path.exists(sisoenv_path):
|
||||
return ""
|
||||
with open(sisoenv_path) as f:
|
||||
for line in f:
|
||||
m = re.match('SISO_PROJECT=\s*(\S*)\s*', line)
|
||||
if m:
|
||||
return m[1]
|
||||
return ""
|
||||
|
||||
|
||||
def _quote_for_cmd(arg):
|
||||
@@ -200,6 +128,7 @@ def _main_inner(input_args, build_id, should_collect_logs=False):
|
||||
offline = False
|
||||
output_dir = "."
|
||||
summarize_build = os.environ.get("NINJA_SUMMARIZE_BUILD") == "1"
|
||||
project = None
|
||||
|
||||
# Ninja uses getopt_long, which allow to intermix non-option arguments.
|
||||
# To leave non supported parameters untouched, we do not use getopt.
|
||||
@@ -217,6 +146,12 @@ def _main_inner(input_args, build_id, should_collect_logs=False):
|
||||
output_dir = arg[2:]
|
||||
elif arg in ("-o", "--offline"):
|
||||
offline = True
|
||||
elif arg in ("--project", "-project"):
|
||||
project = input_args[index + 2]
|
||||
elif arg.startswith("--project="):
|
||||
project = arg[len("--project="):]
|
||||
elif arg.startswith("-project="):
|
||||
project = arg[len("-project="):]
|
||||
elif arg in ("-h", "--help"):
|
||||
print(
|
||||
"autoninja: Use -o/--offline to temporary disable remote execution.",
|
||||
@@ -262,13 +197,43 @@ def _main_inner(input_args, build_id, should_collect_logs=False):
|
||||
use_reclient = use_remoteexec
|
||||
|
||||
if use_remoteexec:
|
||||
if _is_google_corp_machine_using_external_account():
|
||||
if use_reclient:
|
||||
project = _reclient_rbe_project()
|
||||
if not project:
|
||||
print(
|
||||
"You can't use a non-@google.com account (%s and/or %s) on"
|
||||
" a corp machine.\n"
|
||||
"Please login via `gcloud auth login --update-adc` with"
|
||||
" your @google.com account instead.\n" %
|
||||
(_adc_account(), _gcloud_auth_account()),
|
||||
"Can't detect RBE project to use.\n"
|
||||
"Did you setup properly?\n",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
elif use_siso and project is None:
|
||||
# siso runs locally if empty project is given
|
||||
# even if use_remoteexec=true is set.
|
||||
project = _siso_rbe_project()
|
||||
|
||||
if _is_google_corp_machine():
|
||||
# user may login on non-@google.com account on corp,
|
||||
# but need to use @google.com and rbe-chrome-untrusted
|
||||
# on corp machine.
|
||||
if project == 'rbe-chromium-untrusted':
|
||||
print(
|
||||
"You can't use rbe-chromium-untrusted on corp "
|
||||
"machine.\n"
|
||||
"Please use rbe-chrome-untrusted and @google.com "
|
||||
"account instead to build chromium.\n",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
else:
|
||||
# only @google.com is allowed to use rbe-chrome-untrusted
|
||||
# and use @google.com on non-corp machine is not allowed
|
||||
# by corp security policy.
|
||||
if project == 'rbe-chrome-untrusted':
|
||||
print(
|
||||
"You can't use rbe-chrome-untrusted on non-corp "
|
||||
"machine.\n"
|
||||
"Plase use rbe-chromium-untrusted and non-@google.com "
|
||||
"account instead to build chromium.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
@@ -85,6 +85,10 @@ class AutoninjaTest(trial_dir.TestCase):
|
||||
write(os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'),
|
||||
'RBE_v=2')
|
||||
write(os.path.join('buildtools', 'reclient', 'version.txt'), '0.0')
|
||||
write(
|
||||
os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'),
|
||||
'instance=projects/rbe-chromium-untrusted-test/'
|
||||
'instances/default_instance')
|
||||
autoninja.main(['autoninja.py', '-C', out_dir])
|
||||
run_ninja.assert_called_once()
|
||||
args = run_ninja.call_args.args[0]
|
||||
@@ -104,6 +108,8 @@ class AutoninjaTest(trial_dir.TestCase):
|
||||
with mock.patch('siso.main', return_value=0) as siso_main:
|
||||
out_dir = os.path.join('out', 'dir')
|
||||
write(os.path.join(out_dir, 'args.gn'), 'use_siso=true')
|
||||
write(os.path.join('build', 'config', 'siso', '.sisoenv'),
|
||||
'SISO_PROJECT=rbe-chromium-untrusted-test')
|
||||
autoninja.main(['autoninja.py', '-C', out_dir])
|
||||
siso_main.assert_called_once()
|
||||
args = siso_main.call_args.args[0]
|
||||
@@ -129,6 +135,8 @@ class AutoninjaTest(trial_dir.TestCase):
|
||||
'use_siso=true\nuse_remoteexec=true')
|
||||
write(
|
||||
os.path.join('buildtools', 'reclient_cfgs', 'reproxy.cfg'),
|
||||
'instance=projects/rbe-chromium-untrusted-test/'
|
||||
'instances/default_instance\n'
|
||||
'RBE_v=2')
|
||||
write(os.path.join('buildtools', 'reclient', 'version.txt'),
|
||||
'0.0')
|
||||
@@ -143,41 +151,6 @@ class AutoninjaTest(trial_dir.TestCase):
|
||||
['siso', 'ninja', '-project=', '-reapi_instance=', '-C', out_dir])
|
||||
self.assertEqual(reclient_helper_calls[0][1], 'autosiso')
|
||||
|
||||
@parameterized.expand([
|
||||
("non corp machine", False, None, None, None, False),
|
||||
("non corp adc account", True, "foo@chromium.org", None, None, True),
|
||||
("corp adc account", True, "foo@google.com", None, None, False),
|
||||
("non corp gcloud auth account", True, None, "foo@chromium.org", None,
|
||||
True),
|
||||
("corp gcloud auth account", True, None, "foo@google.com", None, False),
|
||||
("non corp luci auth account", True, None, None, "foo@chromium.org",
|
||||
True),
|
||||
("corp luci auth account", True, None, None, "foo@google.com", False),
|
||||
])
|
||||
def test_is_corp_machine_using_external_account(self, _, is_corp,
|
||||
adc_account,
|
||||
gcloud_auth_account,
|
||||
luci_auth_account,
|
||||
expected):
|
||||
for shelve_file in glob.glob(
|
||||
os.path.join(autoninja._SCRIPT_DIR, ".autoninja*")):
|
||||
# Clear cache.
|
||||
os.remove(shelve_file)
|
||||
|
||||
with mock.patch('autoninja._is_google_corp_machine',
|
||||
return_value=is_corp), mock.patch(
|
||||
'autoninja._adc_account',
|
||||
return_value=adc_account), mock.patch(
|
||||
'autoninja._gcloud_auth_account',
|
||||
return_value=gcloud_auth_account), mock.patch(
|
||||
'autoninja._luci_auth_account',
|
||||
return_value=luci_auth_account):
|
||||
self.assertEqual(
|
||||
bool(
|
||||
# pylint: disable=line-too-long
|
||||
autoninja._is_google_corp_machine_using_external_account()),
|
||||
expected)
|
||||
|
||||
@mock.patch('sys.platform', 'win32')
|
||||
def test_print_cmd_windows(self):
|
||||
args = [
|
||||
|
||||
Reference in New Issue
Block a user