Adds SSO auth to gsutil

Code path:
1. plugins.sso_auth is imported, which adds the AuthHandler class to the global state.
2. HasConfiguredCredentials() in gslib/utils.py is called by gsutil, and will return true if "prodaccess" exists on the system, which tells the system that we don't want a no-op auth handler.
3. When a command is called, all the auth handlers are cycled through and sso_auth.SSOAuth is called, which calls a stubby command to emit a gaiamint'ed oauth2 access token, which is then used as the Authorization Header

if --bypass_prodaccess is passed in, then:
1. HasConfiguredCredentials() will bypass the check for prodaccess, as if it didn't exist.
2. plugins.sso_auth does not get imported.
Which will essentially cause gsutil to behave as if this patch never existed.

So the expected behavior is:
=.boto file does not exist, prodaccess exists, but unauthenticated=
Failure: No handler was ready to authenticate. 3 handlers were checked. ['OAuth2Auth', 'HmacAuthV1Handler', 'SSOAuth'] Check your credentials.

=.boto file exists, prodaccess exists, but unauthenticated=
sso_auth will raise NotReadyToAuthenticate, and the .boto file will be used instead

=.boto file exists, prodaccess exists, authenticated=
sso_auth will be run _after_ the default gsutil authenticator, which causes the sso_auth to be used over whatever the default authentication is.

bypass_prodaccess is passed in by default to upload_to_google_storage because we expect people who use upload_to_google_storage to not need prodaccess and have their own boto file already.  Also the sso_auth plugin will only request a readonlyi token, which will not work for uploading.

BUG=258152

Review URL: https://codereview.chromium.org/86123002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@240266 0039d316-1c4b-4281-b951-d872f2087c98

download_from_google_storage.py

@@ -44,12 +44,13 @@ def GetNormalizedPlatform():
 class Gsutil(object):
   """Call gsutil with some predefined settings.  This is a convenience object,
   and is also immutable."""
-  def __init__(self, path, boto_path, timeout=None):
+  def __init__(self, path, boto_path, timeout=None, bypass_prodaccess=False):
     if not os.path.exists(path):
       raise FileNotFoundError('GSUtil not found in %s' % path)
     self.path = path
     self.timeout = timeout
     self.boto_path = boto_path
+    self.bypass_prodaccess = bypass_prodaccess
 
   def get_sub_env(self):
     env = os.environ.copy()
@@ -68,13 +69,19 @@ class Gsutil(object):
     return env
 
   def call(self, *args):
-    return subprocess2.call((sys.executable, self.path) + args,
-                            env=self.get_sub_env(),
-                            timeout=self.timeout)
+    cmd = [sys.executable, self.path]
+    if self.bypass_prodaccess:
+      cmd.append('--bypass_prodaccess')
+    cmd.extend(args)
+    return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout)
 
   def check_call(self, *args):
+    cmd = [sys.executable, self.path]
+    if self.bypass_prodaccess:
+      cmd.append('--bypass_prodaccess')
+    cmd.extend(args)
     ((out, err), code) = subprocess2.communicate(
-        (sys.executable, self.path) + args,
+        cmd,
         stdout=subprocess2.PIPE,
         stderr=subprocess2.PIPE,
         env=self.get_sub_env(),
@@ -343,17 +350,14 @@ def main(args):
   if options.no_auth:
     options.boto = os.devnull
 
-  # Make sure we can find a working instance of gsutil.
+  # Make sure gsutil exists where we expect it to.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
-    gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
+    gsutil = Gsutil(GSUTIL_DEFAULT_PATH,
+                    boto_path=options.boto,
+                    bypass_prodaccess=options.no_auth)
   else:
-    gsutil = None
-    for path in os.environ["PATH"].split(os.pathsep):
-      if os.path.exists(path) and 'gsutil' in os.listdir(path):
-        gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
-    if not gsutil:
-      parser.error('gsutil not found in %s, bad depot_tools checkout?' %
-                  GSUTIL_DEFAULT_PATH)
+    parser.error('gsutil not found in %s, bad depot_tools checkout?' %
+                 GSUTIL_DEFAULT_PATH)
 
   # Passing in -g/--config will run our copy of GSUtil, then quit.
   if options.config:

tests/download_from_google_storage_unittests.py

@@ -75,15 +75,6 @@ class GstoolsUnitTests(unittest.TestCase):
     self.assertEqual(code, 0)
     self.assertEqual(err, '')
 
-  def test_gsutil_version(self):
-    gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None)
-    _, _, err = gsutil.check_call('version')
-    err_lines = err.splitlines()
-    self.assertEqual(err_lines[0], 'gsutil version 3.25')
-    self.assertEqual(
-        err_lines[1],
-        'checksum c9cffb512f467c0aa54880788b9ee6ca (OK)')
-
   def test_get_sha1(self):
     lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt')
     self.assertEqual(

third_party/gsutil/README.chromium

@@ -15,6 +15,9 @@ Modifications:
 * Moved gsutil/third_party into our own third_party directory
 * Append sys.path in gsutil/gsutil to find the moved third_party modules
 * Updated checksum ce71ac982f1148315e7fa65cff2f83e8 -> c9cffb512f467c0aa54880788b9ee6ca
-* Remove code to remove http_proxy before boto.config invocation.
+* Removed code to remove http_proxy before boto.config invocation.
+* Added and imports gsutil/plugins/sso_auth.py to support prodaccess
+  based authentication.
+* Added flag to bypass prodaccess authentication.
 
 Full license is in the COPYING file.

third_party/gsutil/gslib/command.py

@@ -169,7 +169,8 @@ class Command(object):
 
   def __init__(self, command_runner, args, headers, debug, parallel_operations,
                gsutil_bin_dir, boto_lib_dir, config_file_list, gsutil_ver,
-               bucket_storage_uri_class, test_method=None):
+               bucket_storage_uri_class, test_method=None,
+               bypass_prodaccess=True):
     """
     Args:
       command_runner: CommandRunner (for commands built atop other commands).
@@ -186,6 +187,7 @@ class Command(object):
       test_method: Optional general purpose method for testing purposes.
                    Application and semantics of this method will vary by
                    command and test type.
+      bypass_prodaccess: Boolean to ignore the existance of prodaccess.
 
     Implementation note: subclasses shouldn't need to define an __init__
     method, and instead depend on the shared initialization that happens
@@ -209,6 +211,7 @@ class Command(object):
     self.exclude_symlinks = False
     self.recursion_requested = False
     self.all_versions = False
+    self.bypass_prodaccess = bypass_prodaccess
 
     # Process sub-command instance specifications.
     # First, ensure subclass implementation sets all required keys.
@@ -343,9 +346,9 @@ class Command(object):
     if os.path.isfile(acl_arg):
       acl_file = open(acl_arg, 'r')
       acl_arg = acl_file.read()
-      
+
       # TODO: Remove this workaround when GCS allows
-      # whitespace in the Permission element on the server-side 
+      # whitespace in the Permission element on the server-side
       acl_arg = re.sub(r'<Permission>\s*(\S+)\s*</Permission>',
                        r'<Permission>\1</Permission>', acl_arg)
 
@@ -642,7 +645,7 @@ class Command(object):
   def _ConfigureNoOpAuthIfNeeded(self):
     """Sets up no-op auth handler if no boto credentials are configured."""
     config = boto.config
-    if not util.HasConfiguredCredentials():
+    if not util.HasConfiguredCredentials(self.bypass_prodaccess):
       if self.config_file_list:
         if (config.has_option('Credentials', 'gs_oauth2_refresh_token')
             and not HAVE_OAUTH2):

third_party/gsutil/gslib/util.py

@@ -16,6 +16,7 @@
 
 import math
 import re
+import os
 import sys
 import time
 
@@ -61,7 +62,7 @@ class ListingStyle(object):
   LONG_LONG = 'LONG_LONG'
 
 
-def HasConfiguredCredentials():
+def HasConfiguredCredentials(bypass_prodaccess):
   """Determines if boto credential/config file exists."""
   config = boto.config
   has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
@@ -71,8 +72,10 @@ def HasConfiguredCredentials():
   has_oauth_creds = (HAVE_OAUTH2 and
       config.has_option('Credentials', 'gs_oauth2_refresh_token'))
   has_auth_plugins = config.has_option('Plugin', 'plugin_directory')
+  # Pretend prodaccess doesn't exist if --bypass_prodaccess is passed in.
+  has_prodaccess = HasExecutable('prodaccess') and not bypass_prodaccess
   return (has_goog_creds or has_amzn_creds or has_oauth_creds
-          or has_auth_plugins)
+          or has_auth_plugins or has_prodaccess)
 
 
 def _RoundToNearestExponent(num):
@@ -153,3 +156,12 @@ def ExtractErrorDetail(e):
   if detail_start != -1 and detail_end != -1:
     return (exc_name, e.body[detail_start+9:detail_end])
   return (exc_name, None)
+
+
+def HasExecutable(filename):
+  """Determines if an executable is available on the system."""
+  for path in os.environ['PATH'].split(os.pathsep):
+    exe_file = os.path.join(path, filename)
+    if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
+      return True
+  return False

third_party/gsutil/gsutil

@@ -109,6 +109,7 @@ def main():
   if sys.version_info[:3] < (2, 6):
     raise CommandException('gsutil requires Python 2.6 or higher.')
 
+  bypass_prodaccess = False
   config_file_list = _GetBotoConfigFileList()
   command_runner = CommandRunner(gsutil_bin_dir, boto_lib_dir, config_file_list,
                                  gsutil_ver)
@@ -128,9 +129,10 @@ def main():
     boto.config.setbool('Boto', 'https_validate_certificates', True)
 
   try:
-    opts, args = getopt.getopt(sys.argv[1:], 'dDvh:m',
+    opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mb',
                                ['debug', 'detailedDebug', 'version', 'help',
-                                'header', 'multithreaded'])
+                                'header', 'multithreaded',
+                                'bypass_prodaccess'])
   except getopt.GetoptError, e:
     _HandleCommandException(CommandException(e.msg))
   for o, a in opts:
@@ -155,6 +157,8 @@ def main():
       headers[hdr_name] = hdr_val
     if o in ('-m', '--multithreaded'):
       parallel_operations = True
+    if o in ('-b', '--bypass_prodaccess'):
+      bypass_prodaccess = True
   if debug > 1:
     sys.stderr.write(
         '***************************** WARNING *****************************\n'
@@ -186,9 +190,13 @@ def main():
   else:
     command_name = args[0]
 
+  if not bypass_prodaccess:
+    import plugins.sso_auth
+
   return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
                                              args[1:], headers, debug,
-                                             parallel_operations)
+                                             parallel_operations,
+                                             bypass_prodaccess)
 
 
 def _GetBotoConfigFileList():
@@ -244,7 +252,8 @@ def _HandleSigQuit(signal_num, cur_stack_frame):
 
 def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
                                         headers=None, debug=0,
-                                        parallel_operations=False):
+                                        parallel_operations=False,
+                                        bypass_prodaccess=False):
   try:
     # Catch ^C so we can print a brief message instead of the normal Python
     # stack trace.
@@ -283,7 +292,7 @@ def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
     # config file (who might previously have been using gsutil only for
     # accessing publicly readable buckets and objects).
     if e.status == 403:
-      if not HasConfiguredCredentials():
+      if not HasConfiguredCredentials(bypass_prodaccess):
         _OutputAndExit(
             'You are attempting to access protected data with no configured '
             'credentials.\nPlease see '

third_party/gsutil/plugins/sso_auth.py

@@ -0,0 +1,105 @@
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""AuthHandler plugin for gsutil's boto to support LOAS based auth."""
+
+import getpass
+import json
+import os
+import re
+import subprocess
+import time
+import urllib2
+
+from boto.auth_handler import AuthHandler
+from boto.auth_handler import NotReadyToAuthenticate
+
+CMD = ['stubby', '--proto2', 'call', 'blade:sso', 'CorpLogin.Exchange']
+
+STUBBY_CMD = """target: {
+  scope: GAIA_USER
+  name: "%s"
+}
+target_credential: {
+  type: OAUTH2_TOKEN
+  oauth2_attributes: {
+    scope: 'https://www.googleapis.com/auth/devstorage.read_only'
+  }
+}"""
+
+COOKIE_LOCATION = os.path.expanduser('~/.devstore_token')
+
+TOKEN_EXPIRY = 300
+
+
+class SSOAuthError(Exception):
+  pass
+
+
+class SSOAuth(AuthHandler):
+  """SSO based auth handler."""
+
+  capability = ['google-oauth2', 's3']
+
+  def __init__(self, path, config, provider):
+    if provider.name == 'google' and self.has_prodaccess():
+      # If we don't have a loas token, then bypass this auth handler.
+      if subprocess.call('loas_check',
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE):
+        raise NotReadyToAuthenticate()
+    else:
+      raise NotReadyToAuthenticate()
+    self.token = None
+    self.expire = 0
+
+  def GetAccessToken(self):
+    """Returns a valid devstore access token.
+
+    This will return from an in-memory cache if the token is there already,
+    then try a filesystem cache, and then runs a stubby call if none of the
+    caches have a valid token.
+    """
+    if self.token and self.expire > time.time():
+      return self.token
+
+    # Try to retrieve token from filesystem cache.
+    if os.path.exists(COOKIE_LOCATION):
+      last_modified = os.path.getmtime(COOKIE_LOCATION)
+      if time.time() - last_modified < TOKEN_EXPIRY:
+        with open(COOKIE_LOCATION, 'rb') as f:
+          self.token = f.read()
+        self.expire = last_modified + TOKEN_EXPIRY
+        return self.token
+
+    # If the token is not in either caches, or has expired, then fetch token.
+    username = '%s@google.com' % getpass.getuser()
+    proc = subprocess.Popen(CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    out, err = proc.communicate(STUBBY_CMD % username)
+    if proc.returncode:
+      raise SSOAuthError('Stubby returned %d\n%s' % (proc.returncode, err))
+    token_match = re.search(r'oauth2_token: "(.*)"$', out)
+
+    if not token_match:
+      raise SSOAuthError('Oauth2 token not found in %s' % out)
+
+    token = token_match.group(1)
+    self.token = token
+    self.expire = time.time() + TOKEN_EXPIRY
+    with os.fdopen(os.open(COOKIE_LOCATION,
+                           os.O_WRONLY | os.O_CREAT,
+                           0600), 'wb') as f:
+      f.write(token)
+    return token
+
+  def add_auth(self, http_request):
+    http_request.headers['Authorization'] = 'OAuth %s' % self.GetAccessToken()
+
+  @staticmethod
+  def has_prodaccess():
+    for path in os.environ['PATH'].split(os.pathsep):
+      exe_file = os.path.join(path, 'prodaccess')
+      if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
+        return True
+    return False

upload_to_google_storage.py

@@ -234,7 +234,8 @@ def main(args):
 
   # Make sure we can find a working instance of gsutil.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
-    gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
+    gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto,
+                    bypass_prodaccess=True)
   else:
     gsutil = None
     for path in os.environ["PATH"].split(os.pathsep):