chromium_depot_tools/siso.py

#!/usr/bin/env python3
# Copyright 2023 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This script is a wrapper around the siso binary that is pulled to
third_party as part of gclient sync. It will automatically find the siso
binary when run inside a gclient source tree, so users can just type
"siso" on the command line."""

import argparse
import itertools
import os
import platform
import signal
import shlex
import shutil
import sys
from typing import Optional

import build_telemetry
import caffeinate
import gclient_paths


def parse_args(args):
    subcmd = ''
    out_dir = "."
    for i, arg in enumerate(args):
        if not arg.startswith("-") and not subcmd:
            subcmd = arg
            continue
        if arg == "-C":
            out_dir = args[i + 1]
        elif arg.startswith("-C"):
            out_dir = arg[2:]
    return subcmd, out_dir


def check_outdir(subcmd, out_dir):
    ninja_marker = os.path.join(out_dir, ".ninja_deps")
    if os.path.exists(ninja_marker):
        print("depot_tools/siso.py: %s contains Ninja state file.\n"
              "Use `autoninja` to use reclient,\n"
              "or run `gn clean %s` to switch from ninja to siso\n" %
              (out_dir, out_dir),
              file=sys.stderr)
        sys.exit(1)


def apply_metrics_labels(args: list[str]) -> list[str]:
    system_dict = {"Windows": "windows", "Darwin": "mac", "Linux": "linux"}
    user_system = system_dict.get(platform.system(), platform.system())

    # TODO(ovsienko) - add targets to the processing. For this, the Siso needs to understand lists.
    for arg in args[1:]:
        # Respect user provided labels, abort.
        if arg.startswith("--metrics_labels") or arg.startswith(
                "-metrics_labels"):
            return args

    result = []
    result.append("type=developer")
    result.append("tool=siso")
    result.append(f"host_os={user_system}")
    return args + ["--metrics_labels", ",".join(result)]


def apply_telemetry_flags(args: list[str], env: dict[str, str]) -> list[str]:
    telemetry_flags = [
        "enable_cloud_monitoring", "enable_cloud_profiler",
        "enable_cloud_trace", "enable_cloud_logging"
    ]
    # Despite go.dev/issue/68312 being fixed, the issue is still reproducible
    # for googlers. Due to this, the flag is still applied while the
    # issue is being investigated.
    os.environ.setdefault("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")
    flags_to_add = []
    for flag in telemetry_flags:
        found = False
        for arg in args:
            if (arg == f"-{flag}" or arg == f"--{flag}"
                    or arg.startswith(f"-{flag}=")
                    or arg.startswith(f"--{flag}=")):
                found = True
                break
        if not found:
            flags_to_add.append(f"--{flag}")

    # This is a temporary measure as on new siso versions metrics_project
    # gets set the same as project by default.
    # TODO: remove this code after we make sure all clients are using new siso versions.

    parser = argparse.ArgumentParser()

    parser.add_argument("-metrics_project", "--metrics_project")
    parser.add_argument("-project", "--project")
    metrics_env_var = "RBE_metrics_project"
    project_env_var = "SISO_PROJECT"
    # If metrics env variable is set, add flags and return.
    if metrics_env_var in env:
        return args + flags_to_add
    # Check if metrics project is set. If so, then add flags and return.
    known_args, _ = parser.parse_known_args(args)
    if known_args.metrics_project:
        return args + flags_to_add
    if known_args.project:
        return args + flags_to_add + [f"--metrics_project={known_args.project}"]
    if project_env_var in env:
        return args + flags_to_add + [f"--metrics_project={env[project_env_var]}"]
    # Default case - no flags are set, so don't add any
    return args


def load_sisorc(rcfile):
    if not os.path.exists(rcfile):
        return [], {}
    global_flags = []
    subcmd_flags = {}
    with open(rcfile) as file:
        for line in file:
            line = line.strip()
            if line.startswith("#"):
                continue
            args = shlex.split(line)
            if len(args) == 0:
                continue
            if line.startswith("-"):
                global_flags.extend(args)
                continue
            subcmd_flags[args[0]] = args[1:]
    return global_flags, subcmd_flags


def apply_sisorc(global_flags, subcmd_flags, args, subcmd):
    new_args = []
    for arg in args:
        if not new_args:
            new_args.extend(global_flags)
        if arg == subcmd:
            new_args.append(arg)
            new_args.extend(subcmd_flags.get(arg, []))
            continue
        new_args.append(arg)
    return new_args


def _fix_system_limits() -> None:
    # On macOS and most Linux distributions, the default limit of open file
    # descriptors is too low (256 and 1024, respectively).
    # This causes a large j value to result in 'Too many open files' errors.
    # Check whether the limit can be raised to a large enough value. If yes,
    # use `resource.setrlimit` to increase the limit when running ninja.
    if sys.platform in ["darwin", "linux"]:
        import resource
        # Increase the number of allowed open file descriptors to the maximum.
        fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
        if fileno_limit < hard_limit:
            try:
                resource.setrlimit(resource.RLIMIT_NOFILE,
                                   (hard_limit, hard_limit))
            except Exception:
                pass


def _is_google_corp_machine():
    """This assumes that corp machine has gcert binary in known location."""
    return shutil.which("gcert") is not None


def main(args, telemetry_cfg: Optional[build_telemetry.Config] = None):
    # Do not raise KeyboardInterrupt on SIGINT so as to give siso time to run
    # cleanup tasks. Siso will be terminated immediately after the second
    # Ctrl-C.
    original_sigint_handler = signal.getsignal(signal.SIGINT)
    if not telemetry_cfg:
        telemetry_cfg = build_telemetry.load_config()
    should_collect_logs = telemetry_cfg.enabled()

    _fix_system_limits()

    def _ignore(signum, frame):
        try:
            # Call the original signal handler.
            original_sigint_handler(signum, frame)
        except KeyboardInterrupt:
            # Do not reraise KeyboardInterrupt so as to not kill siso too early.
            pass

    signal.signal(signal.SIGINT, _ignore)

    if not sys.platform.startswith('win'):
        signal.signal(signal.SIGTERM, lambda signum, frame: None)

    # On Windows the siso.bat script passes along the arguments enclosed in
    # double quotes. This prevents multiple levels of parsing of the special '^'
    # characters needed when compiling a single file.  When this case is
    # detected, we need to split the argument. This means that arguments
    # containing actual spaces are not supported by siso.bat, but that is not a
    # real limitation.
    if sys.platform.startswith('win') and len(args) == 2:
        args = args[:1] + args[1].split()

    # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly.
    # https://openradar.appspot.com/radar?id=5608755232243712
    #
    # Removing those environment variables to avoid affecting clang's behaviors.
    if sys.platform == 'darwin':
        os.environ.pop("CPATH", None)
        os.environ.pop("LIBRARY_PATH", None)
        os.environ.pop("SDKROOT", None)

    # if user doesn't set PYTHONPYCACHEPREFIX and PYTHONDONTWRITEBYTECODE
    # set PYTHONDONTWRITEBYTECODE=1 not to create many *.pyc in workspace
    # and keep workspace clean.
    if not os.environ.get("PYTHONPYCACHEPREFIX"):
        os.environ.setdefault("PYTHONDONTWRITEBYTECODE", "1")

    environ = os.environ.copy()

    subcmd, out_dir = parse_args(args[1:])

    # Get gclient root + src.
    primary_solution_path = gclient_paths.GetPrimarySolutionPath(out_dir)
    gclient_root_path = gclient_paths.FindGclientRoot(out_dir)
    gclient_src_root_path = None
    if gclient_root_path:
        gclient_src_root_path = os.path.join(gclient_root_path, 'src')

    siso_override_path = os.environ.get('SISO_PATH')
    if siso_override_path:
        print('depot_tools/siso.py: Using Siso binary from SISO_PATH: %s.' %
              siso_override_path,
              file=sys.stderr)
        if not os.path.isfile(siso_override_path):
            print(
                'depot_tools/siso.py: Could not find Siso at provided '
                'SISO_PATH.',
                file=sys.stderr)
            return 1

    for base_path in set(
        [primary_solution_path, gclient_root_path, gclient_src_root_path]):
        if not base_path:
            continue
        env = environ.copy()
        sisoenv_path = os.path.join(base_path, 'build', 'config', 'siso',
                                    '.sisoenv')
        if not os.path.exists(sisoenv_path):
            continue
        with open(sisoenv_path) as f:
            for line in f.readlines():
                k, v = line.rstrip().split('=', 1)
                env[k] = v
        backend_config_dir = os.path.join(base_path, 'build', 'config', 'siso',
                                          'backend_config')
        if os.path.exists(backend_config_dir) and not os.path.exists(
                os.path.join(backend_config_dir, 'backend.star')):
            if _is_google_corp_machine():
                print(
                    'build/config/siso/backend_config/backend.star does not '
                    'exist.\n'
                    'backend.star is configured by gclient hook '
                    'build/config/siso/configure_siso.py.\n'
                    'Make sure `rbe_instance` gclient custom vars is correct.\n'
                    'Did you run `gclient runhooks` ?',
                    file=sys.stderr)
            else:
                print(
                    'build/config/siso/backend_config/backend.star does not '
                    'exist.\n'
                    'See build/config/siso/backend_config/README.md',
                    file=sys.stderr)
            return 1
        global_flags, subcmd_flags = load_sisorc(
            os.path.join(base_path, 'build', 'config', 'siso', '.sisorc'))
        siso_paths = [
            siso_override_path,
            os.path.join(base_path, 'third_party', 'siso', 'cipd',
                         'siso' + gclient_paths.GetExeSuffix()),
            os.path.join(base_path, 'third_party', 'siso',
                         'siso' + gclient_paths.GetExeSuffix()),
        ]
        for siso_path in siso_paths:
            if siso_path and os.path.isfile(siso_path):
                check_outdir(subcmd, out_dir)
                new_args = apply_sisorc(global_flags, subcmd_flags, args[1:],
                                        subcmd)
                if args[1:] != new_args:
                    print('depot_tools/siso.py: %s' % shlex.join(new_args),
                          file=sys.stderr)
                # Add ninja specific flags.
                if args[1] == "ninja":
                    new_args = apply_metrics_labels(new_args)
                    if should_collect_logs:
                        new_args = apply_telemetry_flags(new_args, env)
                return caffeinate.run([siso_path] + new_args, env=env)
        print(
            'depot_tools/siso.py: Could not find siso in third_party/siso '
            'of the current project. Did you run gclient sync?',
            file=sys.stderr)
        return 1
    if siso_override_path:
        return caffeinate.run([siso_override_path] + args[1:])

    print(
        'depot_tools/siso.py: Could not find .sisoenv under build/config/siso '
        'of the current project. Did you run gclient sync?',
        file=sys.stderr)
    return 1


if __name__ == '__main__':
    sys.exit(main(sys.argv))