Move gclient path access to separate module.

gclient_utils.py is a kitchen sink and is for that reason expensive
to import. Move the comparatively cheap and simple path routines
to a new gclient_paths module and use that in gn.py, clang_format.py,
dart_format.py.

(To be able to move FindGclientRoot() to gclient_paths.py,
make it use io.open() instead of FileRead(). FileRead() tries
to paper over invalid utf-8, but that was added for presubmits,
not for .gclient files, so this is hopefully fine.)

Cuts gn.py overhead in half (on my Windows laptop from 0.6s to 0.25s,
still high; on my Mac laptop from 0.1s to 0.05s), and probably helps
the other two too.

Completely remove PathDifference() since it's unused.

Bug: 939959
Change-Id: I6a70f6e4c16062b622fb2df8778e8a598d4cc956
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1512058
Commit-Queue: Nico Weber <thakis@chromium.org>
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>

clang_format.py

@@ -9,7 +9,7 @@ clang-format binaries are pulled down from Google Cloud Storage whenever you
 sync Chrome, to platform-specific locations. This script knows how to locate
 those tools, assuming the script is invoked from inside a Chromium checkout."""
 
-import gclient_utils
+import gclient_paths
 import os
 import subprocess
 import sys
@@ -25,14 +25,14 @@ class NotFoundError(Exception):
 
 def FindClangFormatToolInChromiumTree():
   """Return a path to the clang-format executable, or die trying."""
-  bin_path = gclient_utils.GetBuildtoolsPlatformBinaryPath()
+  bin_path = gclient_paths.GetBuildtoolsPlatformBinaryPath()
   if not bin_path:
     raise NotFoundError(
         'Could not find checkout in any parent of the current path.\n'
         'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium checkout.')
 
   tool_path = os.path.join(bin_path,
-                           'clang-format' + gclient_utils.GetExeSuffix())
+                           'clang-format' + gclient_paths.GetExeSuffix())
   if not os.path.exists(tool_path):
     raise NotFoundError('File does not exist: %s' % tool_path)
   return tool_path
@@ -40,7 +40,7 @@ def FindClangFormatToolInChromiumTree():
 
 def FindClangFormatScriptInChromiumTree(script_name):
   """Return a path to a clang-format helper script, or die trying."""
-  tools_path = gclient_utils.GetBuildtoolsPath()
+  tools_path = gclient_paths.GetBuildtoolsPath()
   if not tools_path:
     raise NotFoundError(
         'Could not find checkout in any parent of the current path.\n',

dart_format.py

@@ -14,7 +14,7 @@ import os
 import subprocess
 import sys
 
-import gclient_utils
+import gclient_paths
 
 class NotFoundError(Exception):
   """A file could not be found."""
@@ -26,7 +26,7 @@ class NotFoundError(Exception):
 
 def FindDartFmtToolInChromiumTree():
   """Return a path to the dartfmt executable, or die trying."""
-  primary_solution_path = gclient_utils.GetPrimarySolutionPath()
+  primary_solution_path = gclient_paths.GetPrimarySolutionPath()
   if not primary_solution_path:
     raise NotFoundError(
         'Could not find checkout in any parent of the current path.')

gclient.py

@@ -102,6 +102,7 @@ import detect_host_arch
 import fix_encoding
 import gclient_eval
 import gclient_scm
+import gclient_paths
 import gclient_utils
 import git_cache
 import metrics
@@ -1459,7 +1460,7 @@ it or fix the checkout.
       if options.verbose:
         print('Looking for %s starting from %s\n' % (
             options.config_filename, os.getcwd()))
-      path = gclient_utils.FindGclientRoot(os.getcwd(), options.config_filename)
+      path = gclient_paths.FindGclientRoot(os.getcwd(), options.config_filename)
       if not path:
         if options.verbose:
           print('Couldn\'t find configuration file.')

gclient_paths.py

@@ -0,0 +1,135 @@
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file is imported by various thin wrappers (around gn, clang-format, ...),
+# so it's meant to import very quickly. To keep it that way don't add more
+# code, and even more importantly don't add more toplevel import statements.
+import os
+
+
+def FindGclientRoot(from_dir, filename='.gclient'):
+  """Tries to find the gclient root."""
+  real_from_dir = os.path.realpath(from_dir)
+  path = real_from_dir
+  while not os.path.exists(os.path.join(path, filename)):
+    split_path = os.path.split(path)
+    if not split_path[1]:
+      return None
+    path = split_path[0]
+
+  # If we did not find the file in the current directory, make sure we are in a
+  # sub directory that is controlled by this configuration.
+  if path != real_from_dir:
+    entries_filename = os.path.join(path, filename + '_entries')
+    if not os.path.exists(entries_filename):
+      import sys
+      # If .gclient_entries does not exist, a previous call to gclient sync
+      # might have failed. In that case, we cannot verify that the .gclient
+      # is the one we want to use. In order to not to cause too much trouble,
+      # just issue a warning and return the path anyway.
+      print >> sys.stderr, ("%s missing, %s file in parent directory %s might "
+          "not be the file you want to use." %
+          (entries_filename, filename, path))
+      return path
+    scope = {}
+    try:
+      import io
+      with io.open(entries_filename, encoding='utf-8') as f:
+        exec(f.read(), scope)
+    except SyntaxError, e:
+      SyntaxErrorToError(filename, e)
+    all_directories = scope['entries'].keys()
+    path_to_check = real_from_dir[len(path)+1:]
+    while path_to_check:
+      if path_to_check in all_directories:
+        return path
+      path_to_check = os.path.dirname(path_to_check)
+    return None
+
+  import logging
+  logging.info('Found gclient root at ' + path)
+  return path
+
+
+def GetPrimarySolutionPath():
+  """Returns the full path to the primary solution. (gclient_root + src)"""
+
+  gclient_root = FindGclientRoot(os.getcwd())
+  if not gclient_root:
+    # Some projects might not use .gclient. Try to see whether we're in a git
+    # checkout.
+    top_dir = [os.getcwd()]
+    def filter_fn(line):
+      repo_root_path = os.path.normpath(line.rstrip('\n'))
+      if os.path.exists(repo_root_path):
+        top_dir[0] = repo_root_path
+    try:
+      CheckCallAndFilter(["git", "rev-parse", "--show-toplevel"],
+                         print_stdout=False, filter_fn=filter_fn)
+    except Exception:
+      pass
+    top_dir = top_dir[0]
+    if os.path.exists(os.path.join(top_dir, 'buildtools')):
+      return top_dir
+    return None
+
+  # Some projects' top directory is not named 'src'.
+  source_dir_name = GetGClientPrimarySolutionName(gclient_root) or 'src'
+  return os.path.join(gclient_root, source_dir_name)
+
+
+def GetBuildtoolsPath():
+  """Returns the full path to the buildtools directory.
+  This is based on the root of the checkout containing the current directory."""
+
+  # Overriding the build tools path by environment is highly unsupported and may
+  # break without warning.  Do not rely on this for anything important.
+  override = os.environ.get('CHROMIUM_BUILDTOOLS_PATH')
+  if override is not None:
+    return override
+
+  primary_solution = GetPrimarySolutionPath()
+  if not primary_solution:
+    return None
+  buildtools_path = os.path.join(primary_solution, 'buildtools')
+  if not os.path.exists(buildtools_path):
+    # Buildtools may be in the gclient root.
+    gclient_root = FindGclientRoot(os.getcwd())
+    buildtools_path = os.path.join(gclient_root, 'buildtools')
+  return buildtools_path
+
+
+def GetBuildtoolsPlatformBinaryPath():
+  """Returns the full path to the binary directory for the current platform."""
+  buildtools_path = GetBuildtoolsPath()
+  if not buildtools_path:
+    return None
+
+  if sys.platform.startswith(('cygwin', 'win')):
+    subdir = 'win'
+  elif sys.platform == 'darwin':
+    subdir = 'mac'
+  elif sys.platform.startswith('linux'):
+      subdir = 'linux64'
+  else:
+    raise Error('Unknown platform: ' + sys.platform)
+  return os.path.join(buildtools_path, subdir)
+
+
+def GetExeSuffix():
+  """Returns '' or '.exe' depending on how executables work on this platform."""
+  if sys.platform.startswith(('cygwin', 'win')):
+    return '.exe'
+  return ''
+
+
+def GetGClientPrimarySolutionName(gclient_root_dir_path):
+  """Returns the name of the primary solution in the .gclient file specified."""
+  gclient_config_file = os.path.join(gclient_root_dir_path, '.gclient')
+  env = {}
+  execfile(gclient_config_file, env)
+  solutions = env.get('solutions', [])
+  if solutions:
+    return solutions[0].get('name')
+  return None

gclient_utils.py

@@ -619,59 +619,6 @@ class GitFilter(object):
     print >> self.out_fh, line
 
 
-def FindGclientRoot(from_dir, filename='.gclient'):
-  """Tries to find the gclient root."""
-  real_from_dir = os.path.realpath(from_dir)
-  path = real_from_dir
-  while not os.path.exists(os.path.join(path, filename)):
-    split_path = os.path.split(path)
-    if not split_path[1]:
-      return None
-    path = split_path[0]
-
-  # If we did not find the file in the current directory, make sure we are in a
-  # sub directory that is controlled by this configuration.
-  if path != real_from_dir:
-    entries_filename = os.path.join(path, filename + '_entries')
-    if not os.path.exists(entries_filename):
-      # If .gclient_entries does not exist, a previous call to gclient sync
-      # might have failed. In that case, we cannot verify that the .gclient
-      # is the one we want to use. In order to not to cause too much trouble,
-      # just issue a warning and return the path anyway.
-      print >> sys.stderr, ("%s missing, %s file in parent directory %s might "
-          "not be the file you want to use." %
-          (entries_filename, filename, path))
-      return path
-    scope = {}
-    try:
-      exec(FileRead(entries_filename), scope)
-    except SyntaxError, e:
-      SyntaxErrorToError(filename, e)
-    all_directories = scope['entries'].keys()
-    path_to_check = real_from_dir[len(path)+1:]
-    while path_to_check:
-      if path_to_check in all_directories:
-        return path
-      path_to_check = os.path.dirname(path_to_check)
-    return None
-
-  logging.info('Found gclient root at ' + path)
-  return path
-
-
-def PathDifference(root, subpath):
-  """Returns the difference subpath minus root."""
-  root = os.path.realpath(root)
-  subpath = os.path.realpath(subpath)
-  if not subpath.startswith(root):
-    return None
-  # If the root does not have a trailing \ or /, we add it so the returned
-  # path starts immediately after the seperator regardless of whether it is
-  # provided.
-  root = os.path.join(root, '')
-  return subpath[len(root):]
-
-
 def FindFileUpwards(filename, path=None):
   """Search upwards from the a directory (default: current) to find a file.
 
@@ -701,89 +648,6 @@ def GetMacWinOrLinux():
   raise Error('Unknown platform: ' + sys.platform)
 
 
-def GetPrimarySolutionPath():
-  """Returns the full path to the primary solution. (gclient_root + src)"""
-
-  gclient_root = FindGclientRoot(os.getcwd())
-  if not gclient_root:
-    # Some projects might not use .gclient. Try to see whether we're in a git
-    # checkout.
-    top_dir = [os.getcwd()]
-    def filter_fn(line):
-      repo_root_path = os.path.normpath(line.rstrip('\n'))
-      if os.path.exists(repo_root_path):
-        top_dir[0] = repo_root_path
-    try:
-      CheckCallAndFilter(["git", "rev-parse", "--show-toplevel"],
-                         print_stdout=False, filter_fn=filter_fn)
-    except Exception:
-      pass
-    top_dir = top_dir[0]
-    if os.path.exists(os.path.join(top_dir, 'buildtools')):
-      return top_dir
-    return None
-
-  # Some projects' top directory is not named 'src'.
-  source_dir_name = GetGClientPrimarySolutionName(gclient_root) or 'src'
-  return os.path.join(gclient_root, source_dir_name)
-
-
-def GetBuildtoolsPath():
-  """Returns the full path to the buildtools directory.
-  This is based on the root of the checkout containing the current directory."""
-
-  # Overriding the build tools path by environment is highly unsupported and may
-  # break without warning.  Do not rely on this for anything important.
-  override = os.environ.get('CHROMIUM_BUILDTOOLS_PATH')
-  if override is not None:
-    return override
-
-  primary_solution = GetPrimarySolutionPath()
-  if not primary_solution:
-    return None
-  buildtools_path = os.path.join(primary_solution, 'buildtools')
-  if not os.path.exists(buildtools_path):
-    # Buildtools may be in the gclient root.
-    gclient_root = FindGclientRoot(os.getcwd())
-    buildtools_path = os.path.join(gclient_root, 'buildtools')
-  return buildtools_path
-
-
-def GetBuildtoolsPlatformBinaryPath():
-  """Returns the full path to the binary directory for the current platform."""
-  buildtools_path = GetBuildtoolsPath()
-  if not buildtools_path:
-    return None
-
-  if sys.platform.startswith(('cygwin', 'win')):
-    subdir = 'win'
-  elif sys.platform == 'darwin':
-    subdir = 'mac'
-  elif sys.platform.startswith('linux'):
-      subdir = 'linux64'
-  else:
-    raise Error('Unknown platform: ' + sys.platform)
-  return os.path.join(buildtools_path, subdir)
-
-
-def GetExeSuffix():
-  """Returns '' or '.exe' depending on how executables work on this platform."""
-  if sys.platform.startswith(('cygwin', 'win')):
-    return '.exe'
-  return ''
-
-
-def GetGClientPrimarySolutionName(gclient_root_dir_path):
-  """Returns the name of the primary solution in the .gclient file specified."""
-  gclient_config_file = os.path.join(gclient_root_dir_path, '.gclient')
-  env = {}
-  execfile(gclient_config_file, env)
-  solutions = env.get('solutions', [])
-  if solutions:
-    return solutions[0].get('name')
-  return None
-
-
 def GetGClientRootAndEntries(path=None):
   """Returns the gclient root and the dict of entries."""
   config_file = '.gclient_entries'

gn.py

@@ -12,7 +12,7 @@ binary. It will also automatically try to find the gn binary when run inside
 the chrome source tree, so users can just type "gn" on the command line
 (normally depot_tools is on the path)."""
 
-import gclient_utils
+import gclient_paths
 import os
 import subprocess
 import sys
@@ -45,22 +45,22 @@ def main(args):
 
   # Try in primary solution location first, with the gn binary having been
   # downloaded by cipd in the projects DEPS.
-  primary_solution_path = gclient_utils.GetPrimarySolutionPath()
+  primary_solution_path = gclient_paths.GetPrimarySolutionPath()
   if primary_solution_path:
     gn_path = os.path.join(primary_solution_path, 'third_party',
-                           'gn', 'gn' + gclient_utils.GetExeSuffix())
+                           'gn', 'gn' + gclient_paths.GetExeSuffix())
     if os.path.exists(gn_path):
       return subprocess.call([gn_path] + args[1:])
 
   # Otherwise try the old .sha1 and download_from_google_storage locations
   # inside of buildtools.
-  bin_path = gclient_utils.GetBuildtoolsPlatformBinaryPath()
+  bin_path = gclient_paths.GetBuildtoolsPlatformBinaryPath()
   if not bin_path:
     print >> sys.stderr, ('gn.py: Could not find checkout in any parent of '
                           'the current path.\nThis must be run inside a '
                           'checkout.')
     return 1
-  gn_path = os.path.join(bin_path, 'gn' + gclient_utils.GetExeSuffix())
+  gn_path = os.path.join(bin_path, 'gn' + gclient_paths.GetExeSuffix())
   if not os.path.exists(gn_path):
     print >> sys.stderr, 'gn.py: Could not find gn executable at: %s' % gn_path
     return 2