InTheForest/chromium_depot_tools
1
0
Fork 0
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools.git synced 2026-01-11 18:51:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reland "gclient: Add commands to edit dependencies and variables in DEPS"

Browse Source 
This is a reland of 7f4c905fc5

Original change's description:
> gclient: Add commands to edit dependencies and variables in DEPS
> 
> Adds 'gclient setvar' and 'gclient setdep' commands to edit variables
> and dependencies in a DEPS file.
> 
> Bug: 760633
> Change-Id: I6c0712cc079dbbbaee6541b7eda71f4b4813b77b
> Reviewed-on: https://chromium-review.googlesource.com/950405
> Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
> Reviewed-by: Aaron Gable <agable@chromium.org>

Bug: 760633
Change-Id: Ia46c74d02e5cc3b67517dfa248f597cb3d98ef3d
Reviewed-on: https://chromium-review.googlesource.com/969457
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>
This commit is contained in:
Edward Lesmes
2018-03-20 17:35:49 -04:00
committed by Commit Bot
parent d3ba72bbc6
commit 6f64a0522b
5 changed files with 366 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
gclient.py
View File

@@ -2859,6 +2859,62 @@ def CMDrevinfo(parser, args):
return 0
def CMDsetdep(parser, args):
parser.add_option('--var', action='append',
dest='vars', metavar='VAR=VAL', default=[],
help='Sets a variable to the given value with the format '
'name=value.')
parser.add_option('-r', '--revision', action='append',
dest='revisions', metavar='DEP@REV', default=[],
help='Sets the revision/version for the dependency with '
'the format dep@rev. If it is a git dependency, dep '
'must be a path and rev must be a git hash or '
'reference (e.g. src/dep@deadbeef). If it is a CIPD '
'dependency, dep must be of the form path:package and '
'rev must be the package version '
'(e.g. src/pkg:chromium/pkg@2.1-cr0).')
parser.add_option('--deps-file', default='DEPS',
# TODO(ehmaldonado): Try to find the DEPS file pointed by
# .gclient first.
help='The DEPS file to be edited. Defaults to the DEPS '
'file in the current directory.')
(options, args) = parser.parse_args(args)
global_scope = {'Var': lambda var: '{%s}' % var}
if not os.path.isfile(options.deps_file):
raise gclient_utils.Error(
'DEPS file %s does not exist.' % options.deps_file)
with open(options.deps_file) as f:
contents = f.read()
local_scope = gclient_eval.Exec(contents, global_scope, {})
for var in options.vars:
name, _, value = var.partition('=')
if not name or not value:
raise gclient_utils.Error(
'Wrong var format: %s should be of the form name=value.' % var)
gclient_eval.SetVar(local_scope, name, value)
for revision in options.revisions:
name, _, value = revision.partition('@')
if not name or not value:
raise gclient_utils.Error(
'Wrong dep format: %s should be of the form dep@rev.' % revision)
if ':' in name:
name, _, package = name.partition(':')
if not name or not package:
raise gclient_utils.Error(
'Wrong CIPD format: %s:%s should be of the form path:pkg@version.'
% (name, package))
gclient_eval.SetCIPD(local_scope, name, package, value)
else:
gclient_eval.SetRevision(local_scope, global_scope, name, value)
with open(options.deps_file, 'w') as f:
f.write(gclient_eval.RenderDEPSFile(local_scope))
def CMDverify(parser, args):
"""Verifies the DEPS file deps are only from allowed_hosts."""
(options, args) = parser.parse_args(args)

197
gclient_eval.py
View File

@@ -3,17 +3,58 @@
# found in the LICENSE file.
import ast
import cStringIO
import collections
import tokenize
from third_party import schema
class _NodeDict(collections.MutableMapping):
"""Dict-like type that also stores information on AST nodes and tokens."""
def __init__(self, data, tokens=None):
self.data = collections.OrderedDict(data)
self.tokens = tokens
def __str__(self):
return str({k: v[0] for k, v in self.data.iteritems()})
def __getitem__(self, key):
return self.data[key][0]
def __setitem__(self, key, value):
self.data[key] = (value, None)
def __delitem__(self, key):
del self.data[key]
def __iter__(self):
return iter(self.data)
def __len__(self):
return len(self.data)
def GetNode(self, key):
return self.data[key][1]
def _SetNode(self, key, value, node):
self.data[key] = (value, node)
def _NodeDictSchema(dict_schema):
"""Validate dict_schema after converting _NodeDict to a regular dict."""
def validate(d):
schema.Schema(dict_schema).validate(dict(d))
return True
return validate
# See https://github.com/keleshev/schema for docs how to configure schema.
_GCLIENT_DEPS_SCHEMA = {
_GCLIENT_DEPS_SCHEMA = _NodeDictSchema({
schema.Optional(basestring): schema.Or(
None,
basestring,
{
_NodeDictSchema({
# Repo and revision to check out under the path
# (same as if no dict was used).
'url': basestring,
@@ -23,25 +64,25 @@ _GCLIENT_DEPS_SCHEMA = {
schema.Optional('condition'): basestring,
schema.Optional('dep_type', default='git'): basestring,
},
}),
# CIPD package.
{
_NodeDictSchema({
'packages': [
{
_NodeDictSchema({
'package': basestring,
'version': basestring,
}
})
],
schema.Optional('condition'): basestring,
schema.Optional('dep_type', default='cipd'): basestring,
},
}),
),
}
})
_GCLIENT_HOOKS_SCHEMA = [{
_GCLIENT_HOOKS_SCHEMA = [_NodeDictSchema({
# Hook action: list of command-line arguments to invoke.
'action': [basestring],
@@ -59,9 +100,9 @@ _GCLIENT_HOOKS_SCHEMA = [{
# Optional condition string. The hook will only be run
# if the condition evaluates to True.
schema.Optional('condition'): basestring,
}]
})]
_GCLIENT_SCHEMA = schema.Schema({
_GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({
# List of host names from which dependencies are allowed (whitelist).
# NOTE: when not present, all hosts are allowed.
# NOTE: scoped to current DEPS file, not recursive.
@@ -79,9 +120,9 @@ _GCLIENT_SCHEMA = schema.Schema({
# Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux').
# Also see 'target_os'.
schema.Optional('deps_os'): {
schema.Optional('deps_os'): _NodeDictSchema({
schema.Optional(basestring): _GCLIENT_DEPS_SCHEMA,
},
}),
# Path to GN args file to write selected variables.
schema.Optional('gclient_gn_args_file'): basestring,
@@ -95,9 +136,9 @@ _GCLIENT_SCHEMA = schema.Schema({
schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA,
# Similar to 'hooks', also keyed by OS.
schema.Optional('hooks_os'): {
schema.Optional('hooks_os'): _NodeDictSchema({
schema.Optional(basestring): _GCLIENT_HOOKS_SCHEMA
},
}),
# Rules which #includes are allowed in the directory.
# Also see 'skip_child_includes' and 'specific_include_rules'.
@@ -123,9 +164,9 @@ _GCLIENT_SCHEMA = schema.Schema({
# Mapping from paths to include rules specific for that path.
# See 'include_rules' for more details.
schema.Optional('specific_include_rules'): {
schema.Optional('specific_include_rules'): _NodeDictSchema({
schema.Optional(basestring): [basestring]
},
}),
# List of additional OS names to consider when selecting dependencies
# from deps_os.
@@ -136,10 +177,10 @@ _GCLIENT_SCHEMA = schema.Schema({
schema.Optional('use_relative_paths'): bool,
# Variables that can be referenced using Var() - see 'deps'.
schema.Optional('vars'): {
schema.Optional('vars'): _NodeDictSchema({
schema.Optional(basestring): schema.Or(basestring, bool),
},
})
}),
}))
def _gclient_eval(node_or_string, global_scope, filename='<unknown>'):
@@ -159,8 +200,7 @@ def _gclient_eval(node_or_string, global_scope, filename='<unknown>'):
elif isinstance(node, ast.List):
return list(map(_convert, node.elts))
elif isinstance(node, ast.Dict):
return collections.OrderedDict(
(_convert(k), _convert(v))
return _NodeDict((_convert(k), (_convert(v), v))
for k, v in zip(node.keys, node.values))
elif isinstance(node, ast.Name):
if node.id not in _allowed_names:
@@ -197,6 +237,7 @@ def Exec(content, global_scope, local_scope, filename='<unknown>'):
if isinstance(node_or_string, ast.Expression):
node_or_string = node_or_string.body
defined_variables = set()
def _visit_in_module(node):
if isinstance(node, ast.Assign):
if len(node.targets) != 1:
@@ -210,12 +251,13 @@ def Exec(content, global_scope, local_scope, filename='<unknown>'):
filename, getattr(node, 'lineno', '<unknown>')))
value = _gclient_eval(node.value, global_scope, filename=filename)
if target.id in local_scope:
if target.id in defined_variables:
raise ValueError(
'invalid assignment: overrides var %r (file %r, line %s)' % (
target.id, filename, getattr(node, 'lineno', '<unknown>')))
local_scope[target.id] = value
defined_variables.add(target.id)
return target.id, (value, node.value)
else:
raise ValueError(
'unexpected AST node: %s %s (file %r, line %s)' % (
@@ -223,8 +265,15 @@ def Exec(content, global_scope, local_scope, filename='<unknown>'):
getattr(node, 'lineno', '<unknown>')))
if isinstance(node_or_string, ast.Module):
data = []
for stmt in node_or_string.body:
_visit_in_module(stmt)
data.append(_visit_in_module(stmt))
tokens = {
token[2]: list(token)
for token in tokenize.generate_tokens(
cStringIO.StringIO(content).readline)
}
local_scope = _NodeDict(data, tokens)
else:
raise ValueError(
'unexpected AST node: %s %s (file %r, line %s)' % (
@@ -333,3 +382,101 @@ def EvaluateCondition(condition, variables, referenced_variables=None):
'unexpected AST node: %s %s (inside %r)' % (
node, ast.dump(node), condition))
return _convert(main_node)
def RenderDEPSFile(gclient_dict):
contents = sorted(gclient_dict.tokens.values(), key=lambda token: token[2])
return tokenize.untokenize(contents)
def _UpdateAstString(tokens, node, value):
position = node.lineno, node.col_offset
tokens[position][1] = repr(value)
node.s = value
def SetVar(gclient_dict, var_name, value):
if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
raise ValueError(
"Can't use SetVar for the given gclient dict. It contains no "
"formatting information.")
tokens = gclient_dict.tokens
if 'vars' not in gclient_dict or var_name not in gclient_dict['vars']:
raise ValueError(
"Could not find any variable called %s." % var_name)
node = gclient_dict['vars'].GetNode(var_name)
if node is None:
raise ValueError(
"The vars entry for %s has no formatting information." % var_name)
_UpdateAstString(tokens, node, value)
gclient_dict['vars']._SetNode(var_name, value, node)
def SetCIPD(gclient_dict, dep_name, package_name, new_version):
if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
raise ValueError(
"Can't use SetCIPD for the given gclient dict. It contains no "
"formatting information.")
tokens = gclient_dict.tokens
if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']:
raise ValueError(
"Could not find any dependency called %s." % dep_name)
# Find the package with the given name
packages = [
package
for package in gclient_dict['deps'][dep_name]['packages']
if package['package'] == package_name
]
if len(packages) != 1:
raise ValueError(
"There must be exactly one package with the given name (%s), "
"%s were found." % (package_name, len(packages)))
# TODO(ehmaldonado): Support Var in package's version.
node = packages[0].GetNode('version')
if node is None:
raise ValueError(
"The deps entry for %s:%s has no formatting information." %
(dep_name, package_name))
new_version = 'version:' + new_version
_UpdateAstString(tokens, node, new_version)
packages[0]._SetNode('version', new_version, node)
def SetRevision(gclient_dict, global_scope, dep_name, new_revision):
if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
raise ValueError(
"Can't use SetRevision for the given gclient dict. It contains no "
"formatting information.")
tokens = gclient_dict.tokens
if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']:
raise ValueError(
"Could not find any dependency called %s." % dep_name)
def _UpdateRevision(dep_dict, dep_key):
dep_node = dep_dict.GetNode(dep_key)
if dep_node is None:
raise ValueError(
"The deps entry for %s has no formatting information." % dep_name)
node = dep_node
if isinstance(node, ast.BinOp):
node = node.right
if isinstance(node, ast.Call):
SetVar(gclient_dict, node.args[0].s, new_revision)
else:
_UpdateAstString(tokens, node, new_revision)
value = _gclient_eval(dep_node, global_scope)
dep_dict._SetNode(dep_key, value, dep_node)
if isinstance(gclient_dict['deps'][dep_name], _NodeDict):
_UpdateRevision(gclient_dict['deps'][dep_name], 'url')
else:
_UpdateRevision(gclient_dict['deps'], dep_name)

2
gclient_utils.py
View File

@@ -1278,7 +1278,7 @@ def freeze(obj):
Will raise TypeError if you pass an object which is not hashable.
"""
if isinstance(obj, dict):
if isinstance(obj, collections.Mapping):
return FrozenDict((freeze(k), freeze(v)) for k, v in obj.iteritems())
elif isinstance(obj, (list, tuple)):
return tuple(freeze(i) for i in obj)

103
tests/gclient_eval_unittest.py
View File

@@ -8,6 +8,7 @@ import itertools
import logging
import os
import sys
import textwrap
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -18,6 +19,45 @@ import gclient
import gclient_eval
_SAMPLE_DEPS_FILE = textwrap.dedent("""\
deps = {
'src/dep': Var('git_repo') + '/dep' + '@' + 'deadbeef',
# Some comment
'src/android/dep_2': {
'url': Var('git_repo') + '/dep_2' + '@' + Var('dep_2_rev'),
'condition': 'checkout_android',
},
'src/dep_3': Var('git_repo') + '/dep_3@' + Var('dep_3_rev'),
'src/cipd/package': {
'packages': [
{
'package': 'some/cipd/package',
'version': 'version:1234',
},
{
'package': 'another/cipd/package',
'version': 'version:5678',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
}
vars = {
'git_repo': 'https://example.com/repo.git',
# Some comment with bad indentation
'dep_2_rev': '1ced',
# Some more comments
# 1
# 2
# 3
'dep_3_rev': '5p1e5',
}""")
class GClientEvalTest(unittest.TestCase):
def test_str(self):
self.assertEqual('foo', gclient_eval._gclient_eval('"foo"', {}))
@@ -82,17 +122,13 @@ class ExecTest(unittest.TestCase):
self.assertIn(
'invalid assignment: overrides var \'a\'', str(cm.exception))
def test_schema_unknown_key(self):
with self.assertRaises(schema.SchemaWrongKeyError):
gclient_eval.Exec('foo = "bar"', {}, {}, '<string>')
def test_schema_wrong_type(self):
with self.assertRaises(schema.SchemaError):
gclient_eval.Exec('include_rules = {}', {}, {}, '<string>')
def test_recursedeps_list(self):
local_scope = {}
gclient_eval.Exec(
local_scope = gclient_eval.Exec(
'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]',
{}, local_scope,
'<string>')
@@ -105,7 +141,7 @@ class ExecTest(unittest.TestCase):
global_scope = {
'Var': lambda var_name: '{%s}' % var_name,
}
gclient_eval.Exec('\n'.join([
local_scope = gclient_eval.Exec('\n'.join([
'vars = {',
' "foo": "bar",',
'}',
@@ -118,6 +154,10 @@ class ExecTest(unittest.TestCase):
'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]),
}, local_scope)
def test_empty_deps(self):
local_scope = gclient_eval.Exec('deps = {}', {}, {}, '<string>')
self.assertEqual({'deps': {}}, local_scope)
class EvaluateConditionTest(unittest.TestCase):
def test_true(self):
@@ -169,6 +209,57 @@ class EvaluateConditionTest(unittest.TestCase):
str(cm.exception))
class SetVarTest(unittest.TestCase):
def testSetVar(self):
local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, {'Var': str}, {})
gclient_eval.SetVar(local_scope, 'dep_2_rev', 'c0ffee')
result = gclient_eval.RenderDEPSFile(local_scope)
self.assertEqual(
result,
_SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee'))
class SetCipdTest(unittest.TestCase):
def testSetCIPD(self):
local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, {'Var': str}, {})
gclient_eval.SetCIPD(
local_scope, 'src/cipd/package', 'another/cipd/package', '6.789')
result = gclient_eval.RenderDEPSFile(local_scope)
self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5678', '6.789'))
class SetRevisionTest(unittest.TestCase):
def setUp(self):
self.global_scope = {'Var': str}
self.local_scope = gclient_eval.Exec(
_SAMPLE_DEPS_FILE, self.global_scope, {})
def testSetRevision(self):
gclient_eval.SetRevision(
self.local_scope, self.global_scope, 'src/dep', 'deadfeed')
result = gclient_eval.RenderDEPSFile(self.local_scope)
self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('deadbeef', 'deadfeed'))
def testSetRevisionInUrl(self):
gclient_eval.SetRevision(
self.local_scope, self.global_scope, 'src/dep_3', '0ff1ce')
result = gclient_eval.RenderDEPSFile(self.local_scope)
self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5p1e5', '0ff1ce'))
def testSetRevisionInVars(self):
gclient_eval.SetRevision(
self.local_scope, self.global_scope, 'src/android/dep_2', 'c0ffee')
result = gclient_eval.RenderDEPSFile(self.local_scope)
self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee'))
if __name__ == '__main__':
level = logging.DEBUG if '-v' in sys.argv else logging.FATAL
logging.basicConfig(

40
tests/gclient_test.py
View File

@@ -1155,6 +1155,46 @@ class GclientTest(trial_dir.TestCase):
finally:
self._get_processed()
def testCreatesCipdDependencies(self):
"""Verifies something."""
write(
'.gclient',
'solutions = [\n'
' { "name": "foo", "url": "svn://example.com/foo",\n'
' "deps_file" : ".DEPS.git",\n'
' },\n'
']')
write(
os.path.join('foo', 'DEPS'),
'vars = {\n'
' "lemur_version": "version:1234",\n'
'}\n'
'deps = {\n'
' "bar": {\n'
' "packages": [{\n'
' "package": "lemur",\n'
' "version": Var("lemur_version"),\n'
' }],\n'
' "dep_type": "cipd",\n'
' }\n'
'}')
options, _ = gclient.OptionParser().parse_args([])
options.validate_syntax = True
obj = gclient.GClient.LoadCurrentConfig(options)
self.assertEquals(1, len(obj.dependencies))
sol = obj.dependencies[0]
sol._condition = 'some_condition'
sol.ParseDepsFile()
self.assertEquals(1, len(sol.dependencies))
dep = sol.dependencies[0]
self.assertIsInstance(dep, gclient.CipdDependency)
self.assertEquals(
'https://chrome-infra-packages.appspot.com/lemur@version:1234',
dep.url)
def testSameDirAllowMultipleCipdDeps(self):
"""Verifies gclient allow multiple cipd deps under same directory."""
parser = gclient.OptionParser()