InTheForest/chromium_depot_tools
1
0
Fork 0
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools.git synced 2026-01-11 18:51:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

switch to 4 space indent

Browse Source 
Leave the recipes/ code at 2 space to match the rest of the recipes
project in other repos.

Reformatted using:
files=( $(
	git ls-tree -r --name-only HEAD | \
		grep -Ev -e '^(third_party|recipes)/' | \
		grep '\.py$';
	git grep -l '#!/usr/bin/env.*python' | grep -v '\.py$'
) )
parallel ./yapf -i -- "${files[@]}"
~/chromiumos/chromite/contrib/reflow_overlong_comments "${files[@]}"

The files that still had strings that were too long were manually
reformatted because they were easy and only a few issues.
autoninja.py
clang_format.py
download_from_google_storage.py
fix_encoding.py
gclient_utils.py
git_cache.py
git_common.py
git_map_branches.py
git_reparent_branch.py
gn.py
my_activity.py
owners_finder.py
presubmit_canned_checks.py
reclient_helper.py
reclientreport.py
roll_dep.py
rustfmt.py
siso.py
split_cl.py
subcommand.py
subprocess2.py
swift_format.py
upload_to_google_storage.py

These files still had lines (strings) that were too long, so the pylint
warnings were suppressed with a TODO.
auth.py
gclient.py
gclient_eval.py
gclient_paths.py
gclient_scm.py
gerrit_util.py
git_cl.py
presubmit_canned_checks.py
presubmit_support.py
scm.py

Change-Id: Ia6535c4f2c48d46b589ec1e791dde6c6b2ea858f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4836379
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Auto-Submit: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
This commit is contained in:
Mike Frysinger
2023-09-06 05:48:55 +00:00
committed by LUCI CQ
parent 677616322a
commit 124bb8e53c
102 changed files with 31372 additions and 29647 deletions
Download Patch File Download Diff File Expand all files Collapse all files

273
reclient_helper.py
View File

@@ -19,65 +19,66 @@ import reclient_metrics
def find_reclient_bin_dir():
tools_path = gclient_paths.GetBuildtoolsPath()
if not tools_path:
return None
tools_path = gclient_paths.GetBuildtoolsPath()
if not tools_path:
return None
reclient_bin_dir = os.path.join(tools_path, 'reclient')
if os.path.isdir(reclient_bin_dir):
return reclient_bin_dir
return None
reclient_bin_dir = os.path.join(tools_path, 'reclient')
if os.path.isdir(reclient_bin_dir):
return reclient_bin_dir
return None
def find_reclient_cfg():
tools_path = gclient_paths.GetBuildtoolsPath()
if not tools_path:
return None
tools_path = gclient_paths.GetBuildtoolsPath()
if not tools_path:
return None
reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg')
if os.path.isfile(reclient_cfg):
return reclient_cfg
return None
reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg')
if os.path.isfile(reclient_cfg):
return reclient_cfg
return None
def run(cmd_args):
if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1':
print(' '.join(cmd_args))
return subprocess.call(cmd_args)
if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1':
print(' '.join(cmd_args))
return subprocess.call(cmd_args)
def start_reproxy(reclient_cfg, reclient_bin_dir):
return run([
os.path.join(reclient_bin_dir,
'bootstrap' + gclient_paths.GetExeSuffix()), '--re_proxy=' +
os.path.join(reclient_bin_dir, 'reproxy' + gclient_paths.GetExeSuffix()),
'--cfg=' + reclient_cfg
])
return run([
os.path.join(reclient_bin_dir,
'bootstrap' + gclient_paths.GetExeSuffix()),
'--re_proxy=' + os.path.join(reclient_bin_dir,
'reproxy' + gclient_paths.GetExeSuffix()),
'--cfg=' + reclient_cfg
])
def stop_reproxy(reclient_cfg, reclient_bin_dir):
return run([
os.path.join(reclient_bin_dir,
'bootstrap' + gclient_paths.GetExeSuffix()), '--shutdown',
'--cfg=' + reclient_cfg
])
return run([
os.path.join(reclient_bin_dir,
'bootstrap' + gclient_paths.GetExeSuffix()), '--shutdown',
'--cfg=' + reclient_cfg
])
def find_ninja_out_dir(args):
# Ninja uses getopt_long, which allows to intermix non-option arguments.
# To leave non supported parameters untouched, we do not use getopt.
for index, arg in enumerate(args[1:]):
if arg == '-C':
# + 1 to get the next argument and +1 because we trimmed off args[0]
return args[index + 2]
if arg.startswith('-C'):
# Support -Cout/Default
return arg[2:]
return '.'
# Ninja uses getopt_long, which allows to intermix non-option arguments.
# To leave non supported parameters untouched, we do not use getopt.
for index, arg in enumerate(args[1:]):
if arg == '-C':
# + 1 to get the next argument and +1 because we trimmed off args[0]
return args[index + 2]
if arg.startswith('-C'):
# Support -Cout/Default
return arg[2:]
return '.'
def find_cache_dir(tmp_dir):
"""Helper to find the correct cache directory for a build.
"""Helper to find the correct cache directory for a build.
tmp_dir should be a build specific temp directory within the out directory.
@@ -86,15 +87,15 @@ def find_cache_dir(tmp_dir):
If this is not called from within a gclient checkout, the cache dir will be:
tmp_dir/cache
"""
gclient_root = gclient_paths.FindGclientRoot(os.getcwd())
if gclient_root:
return os.path.join(gclient_root, '.reproxy_cache',
hashlib.md5(tmp_dir.encode()).hexdigest())
return os.path.join(tmp_dir, 'cache')
gclient_root = gclient_paths.FindGclientRoot(os.getcwd())
if gclient_root:
return os.path.join(gclient_root, '.reproxy_cache',
hashlib.md5(tmp_dir.encode()).hexdigest())
return os.path.join(tmp_dir, 'cache')
def set_reproxy_metrics_flags(tool):
"""Helper to setup metrics collection flags for reproxy.
"""Helper to setup metrics collection flags for reproxy.
The following env vars are set if not already set:
RBE_metrics_project=chromium-reclient-metrics
@@ -103,23 +104,23 @@ def set_reproxy_metrics_flags(tool):
RBE_metrics_labels=source=developer,tool={tool}
RBE_metrics_prefix=go.chromium.org
"""
autoninja_id = os.environ.get("AUTONINJA_BUILD_ID")
if autoninja_id is not None:
os.environ.setdefault("RBE_invocation_id", autoninja_id)
os.environ.setdefault("RBE_metrics_project", "chromium-reclient-metrics")
os.environ.setdefault("RBE_metrics_table", "rbe_metrics.builds")
os.environ.setdefault("RBE_metrics_labels", "source=developer,tool=" + tool)
os.environ.setdefault("RBE_metrics_prefix", "go.chromium.org")
autoninja_id = os.environ.get("AUTONINJA_BUILD_ID")
if autoninja_id is not None:
os.environ.setdefault("RBE_invocation_id", autoninja_id)
os.environ.setdefault("RBE_metrics_project", "chromium-reclient-metrics")
os.environ.setdefault("RBE_metrics_table", "rbe_metrics.builds")
os.environ.setdefault("RBE_metrics_labels", "source=developer,tool=" + tool)
os.environ.setdefault("RBE_metrics_prefix", "go.chromium.org")
def remove_mdproxy_from_path():
os.environ["PATH"] = os.pathsep.join(
d for d in os.environ.get("PATH", "").split(os.pathsep)
if "mdproxy" not in d)
os.environ["PATH"] = os.pathsep.join(
d for d in os.environ.get("PATH", "").split(os.pathsep)
if "mdproxy" not in d)
def set_reproxy_path_flags(out_dir, make_dirs=True):
"""Helper to setup the logs and cache directories for reclient.
"""Helper to setup the logs and cache directories for reclient.
Creates the following directory structure if make_dirs is true:
If in a gclient checkout
@@ -146,98 +147,100 @@ def set_reproxy_path_flags(out_dir, make_dirs=True):
Windows Only:
RBE_server_address=pipe://md5(out_dir/.reproxy_tmp)/reproxy.pipe
"""
tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp'))
log_dir = os.path.join(tmp_dir, 'logs')
racing_dir = os.path.join(tmp_dir, 'racing')
cache_dir = find_cache_dir(tmp_dir)
if make_dirs:
if os.path.exists(log_dir):
try:
# Clear log dir before each build to ensure correct metric aggregation.
shutil.rmtree(log_dir)
except OSError:
print(
"Couldn't clear logs because reproxy did "
"not shutdown after the last build",
file=sys.stderr)
os.makedirs(tmp_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
os.makedirs(cache_dir, exist_ok=True)
os.makedirs(racing_dir, exist_ok=True)
os.environ.setdefault("RBE_output_dir", log_dir)
os.environ.setdefault("RBE_proxy_log_dir", log_dir)
os.environ.setdefault("RBE_log_dir", log_dir)
os.environ.setdefault("RBE_cache_dir", cache_dir)
os.environ.setdefault("RBE_racing_tmp_dir", racing_dir)
if sys.platform.startswith('win'):
pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest()
os.environ.setdefault("RBE_server_address",
"pipe://%s/reproxy.pipe" % pipe_dir)
else:
# unix domain socket has path length limit, so use fixed size path here.
# ref: https://www.man7.org/linux/man-pages/man7/unix.7.html
os.environ.setdefault(
"RBE_server_address", "unix:///tmp/reproxy_%s.sock" %
hashlib.sha256(tmp_dir.encode()).hexdigest())
tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp'))
log_dir = os.path.join(tmp_dir, 'logs')
racing_dir = os.path.join(tmp_dir, 'racing')
cache_dir = find_cache_dir(tmp_dir)
if make_dirs:
if os.path.exists(log_dir):
try:
# Clear log dir before each build to ensure correct metric
# aggregation.
shutil.rmtree(log_dir)
except OSError:
print(
"Couldn't clear logs because reproxy did "
"not shutdown after the last build",
file=sys.stderr)
os.makedirs(tmp_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
os.makedirs(cache_dir, exist_ok=True)
os.makedirs(racing_dir, exist_ok=True)
os.environ.setdefault("RBE_output_dir", log_dir)
os.environ.setdefault("RBE_proxy_log_dir", log_dir)
os.environ.setdefault("RBE_log_dir", log_dir)
os.environ.setdefault("RBE_cache_dir", cache_dir)
os.environ.setdefault("RBE_racing_tmp_dir", racing_dir)
if sys.platform.startswith('win'):
pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest()
os.environ.setdefault("RBE_server_address",
"pipe://%s/reproxy.pipe" % pipe_dir)
else:
# unix domain socket has path length limit, so use fixed size path here.
# ref: https://www.man7.org/linux/man-pages/man7/unix.7.html
os.environ.setdefault(
"RBE_server_address", "unix:///tmp/reproxy_%s.sock" %
hashlib.sha256(tmp_dir.encode()).hexdigest())
def set_racing_defaults():
os.environ.setdefault("RBE_local_resource_fraction", "0.2")
os.environ.setdefault("RBE_racing_bias", "0.95")
os.environ.setdefault("RBE_local_resource_fraction", "0.2")
os.environ.setdefault("RBE_racing_bias", "0.95")
@contextlib.contextmanager
def build_context(argv, tool):
# If use_remoteexec is set, but the reclient binaries or configs don't
# exist, display an error message and stop. Otherwise, the build will
# attempt to run with rewrapper wrapping actions, but will fail with
# possible non-obvious problems.
reclient_bin_dir = find_reclient_bin_dir()
reclient_cfg = find_reclient_cfg()
if reclient_bin_dir is None or reclient_cfg is None:
print(('Build is configured to use reclient but necessary binaries '
"or config files can't be found.\n"
'Please check if `"download_remoteexec_cfg": True` custom var is set'
' in `.gclient`, and run `gclient sync`.'),
file=sys.stderr)
yield 1
return
# If use_remoteexec is set, but the reclient binaries or configs don't
# exist, display an error message and stop. Otherwise, the build will
# attempt to run with rewrapper wrapping actions, but will fail with
# possible non-obvious problems.
reclient_bin_dir = find_reclient_bin_dir()
reclient_cfg = find_reclient_cfg()
if reclient_bin_dir is None or reclient_cfg is None:
print(
'Build is configured to use reclient but necessary binaries '
"or config files can't be found.\n"
'Please check if `"download_remoteexec_cfg": True` custom var is '
'set in `.gclient`, and run `gclient sync`.',
file=sys.stderr)
yield 1
return
ninja_out = find_ninja_out_dir(argv)
ninja_out = find_ninja_out_dir(argv)
try:
set_reproxy_path_flags(ninja_out)
except OSError:
print("Error creating reproxy_tmp in output dir", file=sys.stderr)
yield 1
return
try:
set_reproxy_path_flags(ninja_out)
except OSError:
print("Error creating reproxy_tmp in output dir", file=sys.stderr)
yield 1
return
if reclient_metrics.check_status(ninja_out):
set_reproxy_metrics_flags(tool)
if reclient_metrics.check_status(ninja_out):
set_reproxy_metrics_flags(tool)
if os.environ.get('RBE_instance', None):
print('WARNING: Using RBE_instance=%s\n' %
os.environ.get('RBE_instance', ''))
if os.environ.get('RBE_instance', None):
print('WARNING: Using RBE_instance=%s\n' %
os.environ.get('RBE_instance', ''))
remote_disabled = os.environ.get('RBE_remote_disabled')
if remote_disabled not in ('1', 't', 'T', 'true', 'TRUE', 'True'):
set_racing_defaults()
remote_disabled = os.environ.get('RBE_remote_disabled')
if remote_disabled not in ('1', 't', 'T', 'true', 'TRUE', 'True'):
set_racing_defaults()
# TODO(b/292523514) remove this once a fix is landed in reproxy
remove_mdproxy_from_path()
# TODO(b/292523514) remove this once a fix is landed in reproxy
remove_mdproxy_from_path()
start = time.time()
reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir)
elapsed = time.time() - start
print('%1.3f s to start reproxy' % elapsed)
if reproxy_ret_code != 0:
yield reproxy_ret_code
return
try:
yield
finally:
print("Shutting down reproxy...", file=sys.stderr)
start = time.time()
stop_reproxy(reclient_cfg, reclient_bin_dir)
reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir)
elapsed = time.time() - start
print('%1.3f s to stop reproxy' % elapsed)
print('%1.3f s to start reproxy' % elapsed)
if reproxy_ret_code != 0:
yield reproxy_ret_code
return
try:
yield
finally:
print("Shutting down reproxy...", file=sys.stderr)
start = time.time()
stop_reproxy(reclient_cfg, reclient_bin_dir)
elapsed = time.time() - start
print('%1.3f s to stop reproxy' % elapsed)