mirror of
https://chromium.googlesource.com/chromium/tools/depot_tools.git
synced 2026-01-11 18:51:29 +00:00
switch to 4 space indent
Leave the recipes/ code at 2 space to match the rest of the recipes
project in other repos.
Reformatted using:
files=( $(
git ls-tree -r --name-only HEAD | \
grep -Ev -e '^(third_party|recipes)/' | \
grep '\.py$';
git grep -l '#!/usr/bin/env.*python' | grep -v '\.py$'
) )
parallel ./yapf -i -- "${files[@]}"
~/chromiumos/chromite/contrib/reflow_overlong_comments "${files[@]}"
The files that still had strings that were too long were manually
reformatted because they were easy and only a few issues.
autoninja.py
clang_format.py
download_from_google_storage.py
fix_encoding.py
gclient_utils.py
git_cache.py
git_common.py
git_map_branches.py
git_reparent_branch.py
gn.py
my_activity.py
owners_finder.py
presubmit_canned_checks.py
reclient_helper.py
reclientreport.py
roll_dep.py
rustfmt.py
siso.py
split_cl.py
subcommand.py
subprocess2.py
swift_format.py
upload_to_google_storage.py
These files still had lines (strings) that were too long, so the pylint
warnings were suppressed with a TODO.
auth.py
gclient.py
gclient_eval.py
gclient_paths.py
gclient_scm.py
gerrit_util.py
git_cl.py
presubmit_canned_checks.py
presubmit_support.py
scm.py
Change-Id: Ia6535c4f2c48d46b589ec1e791dde6c6b2ea858f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4836379
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Auto-Submit: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
This commit is contained in:
Mike Frysinger
466
metrics.py
466
metrics.py
@@ -20,7 +20,6 @@ import gclient_utils
|
||||
import metrics_utils
|
||||
import subprocess2
|
||||
|
||||
|
||||
DEPOT_TOOLS = os.path.dirname(os.path.abspath(__file__))
|
||||
CONFIG_FILE = os.path.join(DEPOT_TOOLS, 'metrics.cfg')
|
||||
UPLOAD_SCRIPT = os.path.join(DEPOT_TOOLS, 'upload_metrics.py')
|
||||
@@ -32,294 +31,297 @@ DEPOT_TOOLS_ENV = ['DOGFOOD_STACKED_CHANGES']
|
||||
|
||||
INVALID_CONFIG_WARNING = (
|
||||
'WARNING: Your metrics.cfg file was invalid or nonexistent. A new one will '
|
||||
'be created.'
|
||||
)
|
||||
'be created.')
|
||||
PERMISSION_DENIED_WARNING = (
|
||||
'Could not write the metrics collection config:\n\t%s\n'
|
||||
'Metrics collection will be disabled.'
|
||||
)
|
||||
'Metrics collection will be disabled.')
|
||||
|
||||
|
||||
class _Config(object):
|
||||
def __init__(self):
|
||||
self._initialized = False
|
||||
self._config = {}
|
||||
def __init__(self):
|
||||
self._initialized = False
|
||||
self._config = {}
|
||||
|
||||
def _ensure_initialized(self):
|
||||
if self._initialized:
|
||||
return
|
||||
def _ensure_initialized(self):
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
# Metrics collection is disabled, so don't collect any metrics.
|
||||
if not metrics_utils.COLLECT_METRICS:
|
||||
self._config = {
|
||||
'is-googler': False,
|
||||
'countdown': 0,
|
||||
'opt-in': False,
|
||||
'version': metrics_utils.CURRENT_VERSION,
|
||||
}
|
||||
self._initialized = True
|
||||
return
|
||||
# Metrics collection is disabled, so don't collect any metrics.
|
||||
if not metrics_utils.COLLECT_METRICS:
|
||||
self._config = {
|
||||
'is-googler': False,
|
||||
'countdown': 0,
|
||||
'opt-in': False,
|
||||
'version': metrics_utils.CURRENT_VERSION,
|
||||
}
|
||||
self._initialized = True
|
||||
return
|
||||
|
||||
# We are running on a bot. Ignore config and collect metrics.
|
||||
if metrics_utils.REPORT_BUILD:
|
||||
self._config = {
|
||||
'is-googler': True,
|
||||
'countdown': 0,
|
||||
'opt-in': True,
|
||||
'version': metrics_utils.CURRENT_VERSION,
|
||||
}
|
||||
self._initialized = True
|
||||
return
|
||||
# We are running on a bot. Ignore config and collect metrics.
|
||||
if metrics_utils.REPORT_BUILD:
|
||||
self._config = {
|
||||
'is-googler': True,
|
||||
'countdown': 0,
|
||||
'opt-in': True,
|
||||
'version': metrics_utils.CURRENT_VERSION,
|
||||
}
|
||||
self._initialized = True
|
||||
return
|
||||
|
||||
try:
|
||||
config = json.loads(gclient_utils.FileRead(CONFIG_FILE))
|
||||
except (IOError, ValueError):
|
||||
config = {}
|
||||
try:
|
||||
config = json.loads(gclient_utils.FileRead(CONFIG_FILE))
|
||||
except (IOError, ValueError):
|
||||
config = {}
|
||||
|
||||
self._config = config.copy()
|
||||
self._config = config.copy()
|
||||
|
||||
if 'is-googler' not in self._config:
|
||||
# /should-upload is only accessible from Google IPs, so we only need to
|
||||
# check if we can reach the page. An external developer would get access
|
||||
# denied.
|
||||
try:
|
||||
req = urllib.request.urlopen(metrics_utils.APP_URL + '/should-upload')
|
||||
self._config['is-googler'] = req.getcode() == 200
|
||||
except (urllib.request.URLError, urllib.request.HTTPError):
|
||||
self._config['is-googler'] = False
|
||||
if 'is-googler' not in self._config:
|
||||
# /should-upload is only accessible from Google IPs, so we only need
|
||||
# to check if we can reach the page. An external developer would get
|
||||
# access denied.
|
||||
try:
|
||||
req = urllib.request.urlopen(metrics_utils.APP_URL +
|
||||
'/should-upload')
|
||||
self._config['is-googler'] = req.getcode() == 200
|
||||
except (urllib.request.URLError, urllib.request.HTTPError):
|
||||
self._config['is-googler'] = False
|
||||
|
||||
# Make sure the config variables we need are present, and initialize them to
|
||||
# safe values otherwise.
|
||||
self._config.setdefault('countdown', DEFAULT_COUNTDOWN)
|
||||
self._config.setdefault('opt-in', None)
|
||||
self._config.setdefault('version', metrics_utils.CURRENT_VERSION)
|
||||
# Make sure the config variables we need are present, and initialize
|
||||
# them to safe values otherwise.
|
||||
self._config.setdefault('countdown', DEFAULT_COUNTDOWN)
|
||||
self._config.setdefault('opt-in', None)
|
||||
self._config.setdefault('version', metrics_utils.CURRENT_VERSION)
|
||||
|
||||
if config != self._config:
|
||||
print(INVALID_CONFIG_WARNING, file=sys.stderr)
|
||||
self._write_config()
|
||||
if config != self._config:
|
||||
print(INVALID_CONFIG_WARNING, file=sys.stderr)
|
||||
self._write_config()
|
||||
|
||||
self._initialized = True
|
||||
self._initialized = True
|
||||
|
||||
def _write_config(self):
|
||||
try:
|
||||
gclient_utils.FileWrite(CONFIG_FILE, json.dumps(self._config))
|
||||
except IOError as e:
|
||||
print(PERMISSION_DENIED_WARNING % e, file=sys.stderr)
|
||||
self._config['opt-in'] = False
|
||||
def _write_config(self):
|
||||
try:
|
||||
gclient_utils.FileWrite(CONFIG_FILE, json.dumps(self._config))
|
||||
except IOError as e:
|
||||
print(PERMISSION_DENIED_WARNING % e, file=sys.stderr)
|
||||
self._config['opt-in'] = False
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
self._ensure_initialized()
|
||||
return self._config['version']
|
||||
@property
|
||||
def version(self):
|
||||
self._ensure_initialized()
|
||||
return self._config['version']
|
||||
|
||||
@property
|
||||
def is_googler(self):
|
||||
self._ensure_initialized()
|
||||
return self._config['is-googler']
|
||||
@property
|
||||
def is_googler(self):
|
||||
self._ensure_initialized()
|
||||
return self._config['is-googler']
|
||||
|
||||
@property
|
||||
def opted_in(self):
|
||||
self._ensure_initialized()
|
||||
return self._config['opt-in']
|
||||
@property
|
||||
def opted_in(self):
|
||||
self._ensure_initialized()
|
||||
return self._config['opt-in']
|
||||
|
||||
@opted_in.setter
|
||||
def opted_in(self, value):
|
||||
self._ensure_initialized()
|
||||
self._config['opt-in'] = value
|
||||
self._config['version'] = metrics_utils.CURRENT_VERSION
|
||||
self._write_config()
|
||||
@opted_in.setter
|
||||
def opted_in(self, value):
|
||||
self._ensure_initialized()
|
||||
self._config['opt-in'] = value
|
||||
self._config['version'] = metrics_utils.CURRENT_VERSION
|
||||
self._write_config()
|
||||
|
||||
@property
|
||||
def countdown(self):
|
||||
self._ensure_initialized()
|
||||
return self._config['countdown']
|
||||
@property
|
||||
def countdown(self):
|
||||
self._ensure_initialized()
|
||||
return self._config['countdown']
|
||||
|
||||
@property
|
||||
def should_collect_metrics(self):
|
||||
# Don't report metrics if user is not a Googler.
|
||||
if not self.is_googler:
|
||||
return False
|
||||
# Don't report metrics if user has opted out.
|
||||
if self.opted_in is False:
|
||||
return False
|
||||
# Don't report metrics if countdown hasn't reached 0.
|
||||
if self.opted_in is None and self.countdown > 0:
|
||||
return False
|
||||
return True
|
||||
@property
|
||||
def should_collect_metrics(self):
|
||||
# Don't report metrics if user is not a Googler.
|
||||
if not self.is_googler:
|
||||
return False
|
||||
# Don't report metrics if user has opted out.
|
||||
if self.opted_in is False:
|
||||
return False
|
||||
# Don't report metrics if countdown hasn't reached 0.
|
||||
if self.opted_in is None and self.countdown > 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
def decrease_countdown(self):
|
||||
self._ensure_initialized()
|
||||
if self.countdown == 0:
|
||||
return
|
||||
self._config['countdown'] -= 1
|
||||
if self.countdown == 0:
|
||||
self._config['version'] = metrics_utils.CURRENT_VERSION
|
||||
self._write_config()
|
||||
def decrease_countdown(self):
|
||||
self._ensure_initialized()
|
||||
if self.countdown == 0:
|
||||
return
|
||||
self._config['countdown'] -= 1
|
||||
if self.countdown == 0:
|
||||
self._config['version'] = metrics_utils.CURRENT_VERSION
|
||||
self._write_config()
|
||||
|
||||
def reset_config(self):
|
||||
# Only reset countdown if we're already collecting metrics.
|
||||
if self.should_collect_metrics:
|
||||
self._ensure_initialized()
|
||||
self._config['countdown'] = DEFAULT_COUNTDOWN
|
||||
self._config['opt-in'] = None
|
||||
def reset_config(self):
|
||||
# Only reset countdown if we're already collecting metrics.
|
||||
if self.should_collect_metrics:
|
||||
self._ensure_initialized()
|
||||
self._config['countdown'] = DEFAULT_COUNTDOWN
|
||||
self._config['opt-in'] = None
|
||||
|
||||
|
||||
class MetricsCollector(object):
|
||||
def __init__(self):
|
||||
self._metrics_lock = threading.Lock()
|
||||
self._reported_metrics = {}
|
||||
self._config = _Config()
|
||||
self._collecting_metrics = False
|
||||
self._collect_custom_metrics = True
|
||||
def __init__(self):
|
||||
self._metrics_lock = threading.Lock()
|
||||
self._reported_metrics = {}
|
||||
self._config = _Config()
|
||||
self._collecting_metrics = False
|
||||
self._collect_custom_metrics = True
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self._config
|
||||
@property
|
||||
def config(self):
|
||||
return self._config
|
||||
|
||||
@property
|
||||
def collecting_metrics(self):
|
||||
return self._collecting_metrics
|
||||
@property
|
||||
def collecting_metrics(self):
|
||||
return self._collecting_metrics
|
||||
|
||||
def add(self, name, value):
|
||||
if self._collect_custom_metrics:
|
||||
with self._metrics_lock:
|
||||
self._reported_metrics[name] = value
|
||||
def add(self, name, value):
|
||||
if self._collect_custom_metrics:
|
||||
with self._metrics_lock:
|
||||
self._reported_metrics[name] = value
|
||||
|
||||
def add_repeated(self, name, value):
|
||||
if self._collect_custom_metrics:
|
||||
with self._metrics_lock:
|
||||
self._reported_metrics.setdefault(name, []).append(value)
|
||||
def add_repeated(self, name, value):
|
||||
if self._collect_custom_metrics:
|
||||
with self._metrics_lock:
|
||||
self._reported_metrics.setdefault(name, []).append(value)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def pause_metrics_collection(self):
|
||||
collect_custom_metrics = self._collect_custom_metrics
|
||||
self._collect_custom_metrics = False
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._collect_custom_metrics = collect_custom_metrics
|
||||
@contextlib.contextmanager
|
||||
def pause_metrics_collection(self):
|
||||
collect_custom_metrics = self._collect_custom_metrics
|
||||
self._collect_custom_metrics = False
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._collect_custom_metrics = collect_custom_metrics
|
||||
|
||||
def _upload_metrics_data(self):
|
||||
"""Upload the metrics data to the AppEngine app."""
|
||||
p = subprocess2.Popen(['vpython3', UPLOAD_SCRIPT], stdin=subprocess2.PIPE)
|
||||
# We invoke a subprocess, and use stdin.write instead of communicate(),
|
||||
# so that we are able to return immediately, leaving the upload running in
|
||||
# the background.
|
||||
p.stdin.write(json.dumps(self._reported_metrics).encode('utf-8'))
|
||||
# ... but if we're running on a bot, wait until upload has completed.
|
||||
if metrics_utils.REPORT_BUILD:
|
||||
p.communicate()
|
||||
def _upload_metrics_data(self):
|
||||
"""Upload the metrics data to the AppEngine app."""
|
||||
p = subprocess2.Popen(['vpython3', UPLOAD_SCRIPT],
|
||||
stdin=subprocess2.PIPE)
|
||||
# We invoke a subprocess, and use stdin.write instead of communicate(),
|
||||
# so that we are able to return immediately, leaving the upload running
|
||||
# in the background.
|
||||
p.stdin.write(json.dumps(self._reported_metrics).encode('utf-8'))
|
||||
# ... but if we're running on a bot, wait until upload has completed.
|
||||
if metrics_utils.REPORT_BUILD:
|
||||
p.communicate()
|
||||
|
||||
def _collect_metrics(self, func, command_name, *args, **kwargs):
|
||||
# If we're already collecting metrics, just execute the function.
|
||||
# e.g. git-cl split invokes git-cl upload several times to upload each
|
||||
# split CL.
|
||||
if self.collecting_metrics:
|
||||
# Don't collect metrics for this function.
|
||||
# e.g. Don't record the arguments git-cl split passes to git-cl upload.
|
||||
with self.pause_metrics_collection():
|
||||
return func(*args, **kwargs)
|
||||
def _collect_metrics(self, func, command_name, *args, **kwargs):
|
||||
# If we're already collecting metrics, just execute the function.
|
||||
# e.g. git-cl split invokes git-cl upload several times to upload each
|
||||
# split CL.
|
||||
if self.collecting_metrics:
|
||||
# Don't collect metrics for this function.
|
||||
# e.g. Don't record the arguments git-cl split passes to git-cl
|
||||
# upload.
|
||||
with self.pause_metrics_collection():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
self._collecting_metrics = True
|
||||
self.add('metrics_version', metrics_utils.CURRENT_VERSION)
|
||||
self.add('command', command_name)
|
||||
for env in DEPOT_TOOLS_ENV:
|
||||
if env in os.environ:
|
||||
self.add_repeated('env_vars', {
|
||||
'name': env,
|
||||
'value': os.environ.get(env)
|
||||
})
|
||||
self._collecting_metrics = True
|
||||
self.add('metrics_version', metrics_utils.CURRENT_VERSION)
|
||||
self.add('command', command_name)
|
||||
for env in DEPOT_TOOLS_ENV:
|
||||
if env in os.environ:
|
||||
self.add_repeated('env_vars', {
|
||||
'name': env,
|
||||
'value': os.environ.get(env)
|
||||
})
|
||||
|
||||
try:
|
||||
start = time.time()
|
||||
result = func(*args, **kwargs)
|
||||
exception = None
|
||||
# pylint: disable=bare-except
|
||||
except:
|
||||
exception = sys.exc_info()
|
||||
finally:
|
||||
self.add('execution_time', time.time() - start)
|
||||
try:
|
||||
start = time.time()
|
||||
result = func(*args, **kwargs)
|
||||
exception = None
|
||||
# pylint: disable=bare-except
|
||||
except:
|
||||
exception = sys.exc_info()
|
||||
finally:
|
||||
self.add('execution_time', time.time() - start)
|
||||
|
||||
exit_code = metrics_utils.return_code_from_exception(exception)
|
||||
self.add('exit_code', exit_code)
|
||||
exit_code = metrics_utils.return_code_from_exception(exception)
|
||||
self.add('exit_code', exit_code)
|
||||
|
||||
# Add metrics regarding environment information.
|
||||
self.add('timestamp', int(time.time()))
|
||||
self.add('python_version', metrics_utils.get_python_version())
|
||||
self.add('host_os', gclient_utils.GetOperatingSystem())
|
||||
self.add('host_arch', detect_host_arch.HostArch())
|
||||
# Add metrics regarding environment information.
|
||||
self.add('timestamp', int(time.time()))
|
||||
self.add('python_version', metrics_utils.get_python_version())
|
||||
self.add('host_os', gclient_utils.GetOperatingSystem())
|
||||
self.add('host_arch', detect_host_arch.HostArch())
|
||||
|
||||
depot_tools_age = metrics_utils.get_repo_timestamp(DEPOT_TOOLS)
|
||||
if depot_tools_age is not None:
|
||||
self.add('depot_tools_age', int(depot_tools_age))
|
||||
depot_tools_age = metrics_utils.get_repo_timestamp(DEPOT_TOOLS)
|
||||
if depot_tools_age is not None:
|
||||
self.add('depot_tools_age', int(depot_tools_age))
|
||||
|
||||
git_version = metrics_utils.get_git_version()
|
||||
if git_version:
|
||||
self.add('git_version', git_version)
|
||||
git_version = metrics_utils.get_git_version()
|
||||
if git_version:
|
||||
self.add('git_version', git_version)
|
||||
|
||||
bot_metrics = metrics_utils.get_bot_metrics()
|
||||
if bot_metrics:
|
||||
self.add('bot_metrics', bot_metrics)
|
||||
bot_metrics = metrics_utils.get_bot_metrics()
|
||||
if bot_metrics:
|
||||
self.add('bot_metrics', bot_metrics)
|
||||
|
||||
self._upload_metrics_data()
|
||||
if exception:
|
||||
gclient_utils.reraise(exception[0], exception[1], exception[2])
|
||||
return result
|
||||
self._upload_metrics_data()
|
||||
if exception:
|
||||
gclient_utils.reraise(exception[0], exception[1], exception[2])
|
||||
return result
|
||||
|
||||
def collect_metrics(self, command_name):
|
||||
"""A decorator used to collect metrics over the life of a function.
|
||||
def collect_metrics(self, command_name):
|
||||
"""A decorator used to collect metrics over the life of a function.
|
||||
|
||||
This decorator executes the function and collects metrics about the system
|
||||
environment and the function performance.
|
||||
"""
|
||||
def _decorator(func):
|
||||
if not self.config.should_collect_metrics:
|
||||
return func
|
||||
# Needed to preserve the __name__ and __doc__ attributes of func.
|
||||
@functools.wraps(func)
|
||||
def _inner(*args, **kwargs):
|
||||
return self._collect_metrics(func, command_name, *args, **kwargs)
|
||||
return _inner
|
||||
return _decorator
|
||||
def _decorator(func):
|
||||
if not self.config.should_collect_metrics:
|
||||
return func
|
||||
# Needed to preserve the __name__ and __doc__ attributes of func.
|
||||
@functools.wraps(func)
|
||||
def _inner(*args, **kwargs):
|
||||
return self._collect_metrics(func, command_name, *args,
|
||||
**kwargs)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def print_notice_and_exit(self):
|
||||
"""A context manager used to print the notice and terminate execution.
|
||||
return _inner
|
||||
|
||||
return _decorator
|
||||
|
||||
@contextlib.contextmanager
|
||||
def print_notice_and_exit(self):
|
||||
"""A context manager used to print the notice and terminate execution.
|
||||
|
||||
This decorator executes the function and prints the monitoring notice if
|
||||
necessary. If an exception is raised, we will catch it, and print it before
|
||||
printing the metrics collection notice.
|
||||
This will call sys.exit() with an appropriate exit code to ensure the notice
|
||||
is the last thing printed."""
|
||||
# Needed to preserve the __name__ and __doc__ attributes of func.
|
||||
try:
|
||||
yield
|
||||
exception = None
|
||||
# pylint: disable=bare-except
|
||||
except:
|
||||
exception = sys.exc_info()
|
||||
# Needed to preserve the __name__ and __doc__ attributes of func.
|
||||
try:
|
||||
yield
|
||||
exception = None
|
||||
# pylint: disable=bare-except
|
||||
except:
|
||||
exception = sys.exc_info()
|
||||
|
||||
# Print the exception before the metrics notice, so that the notice is
|
||||
# clearly visible even if gclient fails.
|
||||
if exception:
|
||||
if isinstance(exception[1], KeyboardInterrupt):
|
||||
sys.stderr.write('Interrupted\n')
|
||||
elif not isinstance(exception[1], SystemExit):
|
||||
traceback.print_exception(*exception)
|
||||
# Print the exception before the metrics notice, so that the notice is
|
||||
# clearly visible even if gclient fails.
|
||||
if exception:
|
||||
if isinstance(exception[1], KeyboardInterrupt):
|
||||
sys.stderr.write('Interrupted\n')
|
||||
elif not isinstance(exception[1], SystemExit):
|
||||
traceback.print_exception(*exception)
|
||||
|
||||
# Check if the version has changed
|
||||
if (self.config.is_googler
|
||||
and self.config.opted_in is not False
|
||||
and self.config.version != metrics_utils.CURRENT_VERSION):
|
||||
metrics_utils.print_version_change(self.config.version)
|
||||
self.config.reset_config()
|
||||
# Check if the version has changed
|
||||
if (self.config.is_googler and self.config.opted_in is not False
|
||||
and self.config.version != metrics_utils.CURRENT_VERSION):
|
||||
metrics_utils.print_version_change(self.config.version)
|
||||
self.config.reset_config()
|
||||
|
||||
# Print the notice
|
||||
if self.config.is_googler and self.config.opted_in is None:
|
||||
metrics_utils.print_notice(self.config.countdown)
|
||||
self.config.decrease_countdown()
|
||||
# Print the notice
|
||||
if self.config.is_googler and self.config.opted_in is None:
|
||||
metrics_utils.print_notice(self.config.countdown)
|
||||
self.config.decrease_countdown()
|
||||
|
||||
sys.exit(metrics_utils.return_code_from_exception(exception))
|
||||
sys.exit(metrics_utils.return_code_from_exception(exception))
|
||||
|
||||
|
||||
collector = MetricsCollector()
|
||||
|
||||
Reference in New Issue
Block a user