mirror of
https://chromium.googlesource.com/chromium/tools/depot_tools.git
synced 2026-01-11 10:41:31 +00:00
apply yapf for ninjalog*.py
This is preparation to fix the issue. Bug: 992282 Change-Id: I64b237670bf460ff765df3eaf069b0e0bdcfb31d Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2237268 Commit-Queue: Ye Kuang <yekuang@google.com> Reviewed-by: Ye Kuang <yekuang@google.com> Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
This commit is contained in:
Takuto Ikuta
@@ -2,7 +2,6 @@
|
||||
# Copyright 2018 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""
|
||||
This is script to upload ninja_log from googler.
|
||||
|
||||
@@ -32,79 +31,80 @@ import httplib2
|
||||
# These build configs affect build performance a lot.
|
||||
# TODO(tikuta): Add 'blink_symbol_level', 'closure_compile' and
|
||||
# 'use_jumbo_build'.
|
||||
WHITELISTED_CONFIGS = (
|
||||
'symbol_level', 'use_goma', 'is_debug', 'is_component_build', 'enable_nacl',
|
||||
'host_os', 'host_cpu', 'target_os', 'target_cpu'
|
||||
)
|
||||
WHITELISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug',
|
||||
'is_component_build', 'enable_nacl', 'host_os',
|
||||
'host_cpu', 'target_os', 'target_cpu')
|
||||
|
||||
|
||||
def IsGoogler(server):
|
||||
"""Check whether this script run inside corp network."""
|
||||
try:
|
||||
h = httplib2.Http()
|
||||
_, content = h.request('https://'+server+'/should-upload', 'GET')
|
||||
return content == 'Success'
|
||||
except httplib2.HttpLib2Error:
|
||||
return False
|
||||
"""Check whether this script run inside corp network."""
|
||||
try:
|
||||
h = httplib2.Http()
|
||||
_, content = h.request('https://' + server + '/should-upload', 'GET')
|
||||
return content == 'Success'
|
||||
except httplib2.HttpLib2Error:
|
||||
return False
|
||||
|
||||
|
||||
def ParseGNArgs(gn_args):
|
||||
"""Parse gn_args as json and return config dictionary."""
|
||||
configs = json.loads(gn_args)
|
||||
build_configs = {}
|
||||
"""Parse gn_args as json and return config dictionary."""
|
||||
configs = json.loads(gn_args)
|
||||
build_configs = {}
|
||||
|
||||
for config in configs:
|
||||
key = config["name"]
|
||||
if key not in WHITELISTED_CONFIGS:
|
||||
continue
|
||||
if 'current' in config:
|
||||
build_configs[key] = config['current']['value']
|
||||
else:
|
||||
build_configs[key] = config['default']['value']
|
||||
for config in configs:
|
||||
key = config["name"]
|
||||
if key not in WHITELISTED_CONFIGS:
|
||||
continue
|
||||
if 'current' in config:
|
||||
build_configs[key] = config['current']['value']
|
||||
else:
|
||||
build_configs[key] = config['default']['value']
|
||||
|
||||
return build_configs
|
||||
|
||||
return build_configs
|
||||
|
||||
def GetBuildTargetFromCommandLine(cmdline):
|
||||
"""Get build targets from commandline."""
|
||||
"""Get build targets from commandline."""
|
||||
|
||||
# Skip argv0.
|
||||
idx = 1
|
||||
# Skip argv0.
|
||||
idx = 1
|
||||
|
||||
# Skipping all args that involve these flags, and taking all remaining args
|
||||
# as targets.
|
||||
onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w')
|
||||
zeroarg_flags = ('--version', '-n', '-v')
|
||||
# Skipping all args that involve these flags, and taking all remaining args
|
||||
# as targets.
|
||||
onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w')
|
||||
zeroarg_flags = ('--version', '-n', '-v')
|
||||
|
||||
targets = []
|
||||
targets = []
|
||||
|
||||
while idx < len(cmdline):
|
||||
if cmdline[idx] in onearg_flags:
|
||||
idx += 2
|
||||
continue
|
||||
while idx < len(cmdline):
|
||||
if cmdline[idx] in onearg_flags:
|
||||
idx += 2
|
||||
continue
|
||||
|
||||
if (cmdline[idx][:2] in onearg_flags or
|
||||
cmdline[idx] in zeroarg_flags):
|
||||
idx += 1
|
||||
continue
|
||||
if (cmdline[idx][:2] in onearg_flags or cmdline[idx] in zeroarg_flags):
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
targets.append(cmdline[idx])
|
||||
idx += 1
|
||||
targets.append(cmdline[idx])
|
||||
idx += 1
|
||||
|
||||
return targets
|
||||
|
||||
return targets
|
||||
|
||||
def GetJflag(cmdline):
|
||||
"""Parse cmdline to get flag value for -j"""
|
||||
"""Parse cmdline to get flag value for -j"""
|
||||
|
||||
for i in range(len(cmdline)):
|
||||
if (cmdline[i] == '-j' and i + 1 < len(cmdline) and
|
||||
cmdline[i+1].isdigit()):
|
||||
return int(cmdline[i+1])
|
||||
for i in range(len(cmdline)):
|
||||
if (cmdline[i] == '-j' and i + 1 < len(cmdline)
|
||||
and cmdline[i + 1].isdigit()):
|
||||
return int(cmdline[i + 1])
|
||||
|
||||
if (cmdline[i].startswith('-j') and
|
||||
cmdline[i][len('-j'):].isdigit()):
|
||||
return int(cmdline[i][len('-j'):])
|
||||
if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()):
|
||||
return int(cmdline[i][len('-j'):])
|
||||
|
||||
|
||||
def GetMetadata(cmdline, ninjalog):
|
||||
"""Get metadata for uploaded ninjalog.
|
||||
"""Get metadata for uploaded ninjalog.
|
||||
|
||||
Returned metadata has schema defined in
|
||||
https://cs.chromium.org?q="type+Metadata+struct+%7B"+file:%5Einfra/go/src/infra/appengine/chromium_build_stats/ninjalog/
|
||||
@@ -112,116 +112,122 @@ def GetMetadata(cmdline, ninjalog):
|
||||
TODO(tikuta): Collect GOMA_* env var.
|
||||
"""
|
||||
|
||||
build_dir = os.path.dirname(ninjalog)
|
||||
build_dir = os.path.dirname(ninjalog)
|
||||
|
||||
build_configs = {}
|
||||
|
||||
try:
|
||||
args = ['gn', 'args', build_dir, '--list', '--short', '--json']
|
||||
if sys.platform == 'win32':
|
||||
# gn in PATH is bat file in windows environment (except cygwin).
|
||||
args = ['cmd', '/c'] + args
|
||||
|
||||
gn_args = subprocess.check_output(args)
|
||||
build_configs = ParseGNArgs(gn_args)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logging.error("Failed to call gn %s", e)
|
||||
build_configs = {}
|
||||
|
||||
try:
|
||||
args = ['gn', 'args', build_dir, '--list', '--short', '--json']
|
||||
if sys.platform == 'win32':
|
||||
# gn in PATH is bat file in windows environment (except cygwin).
|
||||
args = ['cmd', '/c'] + args
|
||||
# Stringify config.
|
||||
for k in build_configs:
|
||||
build_configs[k] = str(build_configs[k])
|
||||
|
||||
gn_args = subprocess.check_output(args)
|
||||
build_configs = ParseGNArgs(gn_args)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logging.error("Failed to call gn %s", e)
|
||||
build_configs = {}
|
||||
metadata = {
|
||||
'platform': platform.system(),
|
||||
'cpu_core': multiprocessing.cpu_count(),
|
||||
'build_configs': build_configs,
|
||||
'targets': GetBuildTargetFromCommandLine(cmdline),
|
||||
}
|
||||
|
||||
# Stringify config.
|
||||
for k in build_configs:
|
||||
build_configs[k] = str(build_configs[k])
|
||||
jflag = GetJflag(cmdline)
|
||||
if jflag is not None:
|
||||
metadata['jobs'] = jflag
|
||||
|
||||
metadata = {
|
||||
'platform': platform.system(),
|
||||
'cpu_core': multiprocessing.cpu_count(),
|
||||
'build_configs': build_configs,
|
||||
'targets': GetBuildTargetFromCommandLine(cmdline),
|
||||
}
|
||||
return metadata
|
||||
|
||||
jflag = GetJflag(cmdline)
|
||||
if jflag is not None:
|
||||
metadata['jobs'] = jflag
|
||||
|
||||
return metadata
|
||||
|
||||
def GetNinjalog(cmdline):
|
||||
"""GetNinjalog returns the path to ninjalog from cmdline."""
|
||||
# ninjalog is in current working directory by default.
|
||||
ninjalog_dir = '.'
|
||||
"""GetNinjalog returns the path to ninjalog from cmdline."""
|
||||
# ninjalog is in current working directory by default.
|
||||
ninjalog_dir = '.'
|
||||
|
||||
i = 0
|
||||
while i < len(cmdline):
|
||||
cmd = cmdline[i]
|
||||
i += 1
|
||||
if cmd == '-C' and i < len(cmdline):
|
||||
ninjalog_dir = cmdline[i]
|
||||
i += 1
|
||||
continue
|
||||
i = 0
|
||||
while i < len(cmdline):
|
||||
cmd = cmdline[i]
|
||||
i += 1
|
||||
if cmd == '-C' and i < len(cmdline):
|
||||
ninjalog_dir = cmdline[i]
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if cmd.startswith('-C') and len(cmd) > len('-C'):
|
||||
ninjalog_dir = cmd[len('-C'):]
|
||||
if cmd.startswith('-C') and len(cmd) > len('-C'):
|
||||
ninjalog_dir = cmd[len('-C'):]
|
||||
|
||||
return os.path.join(ninjalog_dir, '.ninja_log')
|
||||
|
||||
return os.path.join(ninjalog_dir, '.ninja_log')
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--server',
|
||||
default='chromium-build-stats.appspot.com',
|
||||
help='server to upload ninjalog file.')
|
||||
parser.add_argument('--ninjalog', help='ninjalog file to upload.')
|
||||
parser.add_argument('--verbose', action='store_true',
|
||||
help='Enable verbose logging.')
|
||||
parser.add_argument('--cmdline', required=True, nargs=argparse.REMAINDER,
|
||||
help='command line args passed to ninja.')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--server',
|
||||
default='chromium-build-stats.appspot.com',
|
||||
help='server to upload ninjalog file.')
|
||||
parser.add_argument('--ninjalog', help='ninjalog file to upload.')
|
||||
parser.add_argument('--verbose',
|
||||
action='store_true',
|
||||
help='Enable verbose logging.')
|
||||
parser.add_argument('--cmdline',
|
||||
required=True,
|
||||
nargs=argparse.REMAINDER,
|
||||
help='command line args passed to ninja.')
|
||||
|
||||
args = parser.parse_args()
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
else:
|
||||
# Disable logging.
|
||||
logging.disable(logging.CRITICAL)
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
else:
|
||||
# Disable logging.
|
||||
logging.disable(logging.CRITICAL)
|
||||
|
||||
if not IsGoogler(args.server):
|
||||
return 0
|
||||
|
||||
|
||||
ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
|
||||
if not os.path.isfile(ninjalog):
|
||||
logging.warn("ninjalog is not found in %s", ninjalog)
|
||||
return 1
|
||||
|
||||
# We assume that each ninja invocation interval takes at least 2 seconds.
|
||||
# This is not to have duplicate entry in server when current build is no-op.
|
||||
if os.stat(ninjalog).st_mtime < time.time() - 2:
|
||||
logging.info("ninjalog is not updated recently %s", ninjalog)
|
||||
return 0
|
||||
|
||||
output = cStringIO.StringIO()
|
||||
|
||||
with open(ninjalog) as f:
|
||||
with gzip.GzipFile(fileobj=output, mode='wb') as g:
|
||||
g.write(f.read())
|
||||
g.write('# end of ninja log\n')
|
||||
|
||||
metadata = GetMetadata(args.cmdline, ninjalog)
|
||||
logging.info('send metadata: %s', json.dumps(metadata))
|
||||
g.write(json.dumps(metadata))
|
||||
|
||||
h = httplib2.Http()
|
||||
resp_headers, content = h.request(
|
||||
'https://'+args.server+'/upload_ninja_log/', 'POST',
|
||||
body=output.getvalue(), headers={'Content-Encoding': 'gzip'})
|
||||
|
||||
if resp_headers.status != 200:
|
||||
logging.warn("unexpected status code for response: %s",
|
||||
resp_headers.status)
|
||||
return 1
|
||||
|
||||
logging.info('response header: %s', resp_headers)
|
||||
logging.info('response content: %s', content)
|
||||
if not IsGoogler(args.server):
|
||||
return 0
|
||||
|
||||
ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
|
||||
if not os.path.isfile(ninjalog):
|
||||
logging.warn("ninjalog is not found in %s", ninjalog)
|
||||
return 1
|
||||
|
||||
# We assume that each ninja invocation interval takes at least 2 seconds.
|
||||
# This is not to have duplicate entry in server when current build is no-op.
|
||||
if os.stat(ninjalog).st_mtime < time.time() - 2:
|
||||
logging.info("ninjalog is not updated recently %s", ninjalog)
|
||||
return 0
|
||||
|
||||
output = cStringIO.StringIO()
|
||||
|
||||
with open(ninjalog) as f:
|
||||
with gzip.GzipFile(fileobj=output, mode='wb') as g:
|
||||
g.write(f.read())
|
||||
g.write('# end of ninja log\n')
|
||||
|
||||
metadata = GetMetadata(args.cmdline, ninjalog)
|
||||
logging.info('send metadata: %s', json.dumps(metadata))
|
||||
g.write(json.dumps(metadata))
|
||||
|
||||
h = httplib2.Http()
|
||||
resp_headers, content = h.request('https://' + args.server +
|
||||
'/upload_ninja_log/',
|
||||
'POST',
|
||||
body=output.getvalue(),
|
||||
headers={'Content-Encoding': 'gzip'})
|
||||
|
||||
if resp_headers.status != 200:
|
||||
logging.warn("unexpected status code for response: %s", resp_headers.status)
|
||||
return 1
|
||||
|
||||
logging.info('response header: %s', resp_headers)
|
||||
logging.info('response content: %s', content)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
sys.exit(main())
|
||||
|
||||
@@ -19,30 +19,30 @@ VERSION = 2
|
||||
|
||||
|
||||
def LoadConfig():
|
||||
if os.path.isfile(CONFIG):
|
||||
with open(CONFIG, 'rb') as f:
|
||||
config = json.load(f)
|
||||
if config['version'] == VERSION:
|
||||
config['countdown'] = max(0, config['countdown'] - 1)
|
||||
return config
|
||||
if os.path.isfile(CONFIG):
|
||||
with open(CONFIG, 'rb') as f:
|
||||
config = json.load(f)
|
||||
if config['version'] == VERSION:
|
||||
config['countdown'] = max(0, config['countdown'] - 1)
|
||||
return config
|
||||
|
||||
return {
|
||||
'is-googler': ninjalog_uploader.IsGoogler(
|
||||
'chromium-build-stats.appspot.com'),
|
||||
'countdown': 10,
|
||||
'version': VERSION,
|
||||
}
|
||||
return {
|
||||
'is-googler':
|
||||
ninjalog_uploader.IsGoogler('chromium-build-stats.appspot.com'),
|
||||
'countdown': 10,
|
||||
'version': VERSION,
|
||||
}
|
||||
|
||||
|
||||
def SaveConfig(config):
|
||||
with open(CONFIG, 'wb') as f:
|
||||
json.dump(config, f)
|
||||
with open(CONFIG, 'wb') as f:
|
||||
json.dump(config, f)
|
||||
|
||||
|
||||
def ShowMessage(countdown):
|
||||
whitelisted = '\n'.join([' * %s' % config for config in
|
||||
ninjalog_uploader.WHITELISTED_CONFIGS])
|
||||
print("""
|
||||
whitelisted = '\n'.join(
|
||||
[' * %s' % config for config in ninjalog_uploader.WHITELISTED_CONFIGS])
|
||||
print("""
|
||||
Your ninjalog will be uploaded to build stats server. The uploaded log will be
|
||||
used to analyze user side build performance.
|
||||
|
||||
@@ -73,46 +73,47 @@ You can find a more detailed explanation in
|
||||
|
||||
|
||||
def main():
|
||||
config = LoadConfig()
|
||||
config = LoadConfig()
|
||||
|
||||
if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
|
||||
config['opt-in'] = True
|
||||
config['countdown'] = 0
|
||||
SaveConfig(config)
|
||||
print('ninjalog upload is opted in.')
|
||||
return 0
|
||||
if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
|
||||
config['opt-in'] = True
|
||||
config['countdown'] = 0
|
||||
SaveConfig(config)
|
||||
print('ninjalog upload is opted in.')
|
||||
return 0
|
||||
|
||||
if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
|
||||
config['opt-in'] = False
|
||||
SaveConfig(config)
|
||||
print('ninjalog upload is opted out.')
|
||||
return 0
|
||||
if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
|
||||
config['opt-in'] = False
|
||||
SaveConfig(config)
|
||||
print('ninjalog upload is opted out.')
|
||||
return 0
|
||||
|
||||
if 'opt-in' in config and not config['opt-in']:
|
||||
# Upload is opted out.
|
||||
return 0
|
||||
if 'opt-in' in config and not config['opt-in']:
|
||||
# Upload is opted out.
|
||||
return 0
|
||||
|
||||
if not config.get("is-googler", False):
|
||||
# Not googler.
|
||||
return 0
|
||||
if not config.get("is-googler", False):
|
||||
# Not googler.
|
||||
return 0
|
||||
|
||||
if config.get("countdown", 0) > 0:
|
||||
# Need to show message.
|
||||
ShowMessage(config["countdown"])
|
||||
# Only save config if something has meaningfully changed.
|
||||
SaveConfig(config)
|
||||
return 0
|
||||
if config.get("countdown", 0) > 0:
|
||||
# Need to show message.
|
||||
ShowMessage(config["countdown"])
|
||||
# Only save config if something has meaningfully changed.
|
||||
SaveConfig(config)
|
||||
return 0
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
# dry-run for debugging.
|
||||
print("upload ninjalog dry-run")
|
||||
return 0
|
||||
if len(sys.argv) == 1:
|
||||
# dry-run for debugging.
|
||||
print("upload ninjalog dry-run")
|
||||
return 0
|
||||
|
||||
# Run upload script without wait.
|
||||
devnull = open(os.devnull, "w")
|
||||
subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:],
|
||||
stdout=devnull, stderr=devnull)
|
||||
# Run upload script without wait.
|
||||
devnull = open(os.devnull, "w")
|
||||
subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:],
|
||||
stdout=devnull,
|
||||
stderr=devnull)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
sys.exit(main())
|
||||
|
||||
Reference in New Issue
Block a user