InTheForest/chromium_depot_tools
1
0
Fork 0
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools.git synced 2026-01-11 18:51:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reland "Make gclient ready for the Blink (DEPS to main project)"

Browse Source 
Reland crrev.com/743083002, which was reverted in crrev.com/796053002
due to some test flakiness, probably related with an old version of Git on
the bots. Relanding now that the infra has been updated to Trusty (plus
adding some de-flake precautions).

Original CL Description:
Make gclient ready for the Blink (DEPS to main project) transition

This CL makes gclient understand correctly whether a git project is
being moved from DEPS to an upper project and vice-versa.
The driving use case for this is the upcoming Blink merge, where
third_party/Webkit will be removed from DEPS (and .gitignore) and will
become part of the main project.

At present state, gclient leaves the .git folder around when a project
is removed from DEPS, and that causes many problems.

Furthermore this CL solves the performance problem of bisecting across
the merge point. The subproject's (Blink) .git/ folder is moved to a
backup location (in the main checkout root) and is restored when moving
backwards, avoiding a re-fetch when bisecting across the merge point.

BUG=431469

Review URL: https://codereview.chromium.org/910913003

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@294082 0039d316-1c4b-4281-b951-d872f2087c98
This commit is contained in:
primiano@chromium.org
2015-02-17 11:15:40 +00:00
parent 27386ddd72
commit 1c1273893e
7 changed files with 243 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
gclient.py
View File

@@ -1542,23 +1542,18 @@ been automagically updated. The previous version is available at %s.old.
# Fix path separator on Windows.
entry_fixed = entry.replace('/', os.path.sep)
e_dir = os.path.join(self.root_dir, entry_fixed)
def _IsParentOfAny(parent, path_list):
parent_plus_slash = parent + '/'
return any(
path[:len(parent_plus_slash)] == parent_plus_slash
for path in path_list)
# Use entry and not entry_fixed there.
if (entry not in entries and
(not any(path.startswith(entry + '/') for path in entries)) and
os.path.exists(e_dir)):
# The entry has been removed from DEPS.
scm = gclient_scm.CreateSCM(
prev_url, self.root_dir, entry_fixed, self.outbuf)
# Check to see if this directory is now part of a higher-up checkout.
# The directory might be part of a git OR svn checkout.
scm_root = None
scm_class = None
for scm_class in (gclient_scm.scm.GIT, gclient_scm.scm.SVN):
try:
scm_root = scm_class.GetCheckoutRoot(scm.checkout_path)
@@ -1571,9 +1566,45 @@ been automagically updated. The previous version is available at %s.old.
'determine whether it is part of a higher-level '
'checkout, so not removing.' % entry)
continue
# This is to handle the case of third_party/WebKit migrating from
# being a DEPS entry to being part of the main project.
# If the subproject is a Git project, we need to remove its .git
# folder. Otherwise git operations on that folder will have different
# effects depending on the current working directory.
if scm_class == gclient_scm.scm.GIT and (
os.path.abspath(scm_root) == os.path.abspath(e_dir)):
e_par_dir = os.path.join(e_dir, os.pardir)
if scm_class.IsInsideWorkTree(e_par_dir):
par_scm_root = scm_class.GetCheckoutRoot(e_par_dir)
# rel_e_dir : relative path of entry w.r.t. its parent repo.
rel_e_dir = os.path.relpath(e_dir, par_scm_root)
if scm_class.IsDirectoryVersioned(par_scm_root, rel_e_dir):
save_dir = scm.GetGitBackupDirPath()
# Remove any eventual stale backup dir for the same project.
if os.path.exists(save_dir):
gclient_utils.rmtree(save_dir)
os.rename(os.path.join(e_dir, '.git'), save_dir)
# When switching between the two states (entry/ is a subproject
# -> entry/ is part of the outer project), it is very likely
# that some files are changed in the checkout, unless we are
# jumping *exactly* across the commit which changed just DEPS.
# In such case we want to cleanup any eventual stale files
# (coming from the old subproject) in order to end up with a
# clean checkout.
scm_class.CleanupDir(par_scm_root, rel_e_dir)
assert not os.path.exists(os.path.join(e_dir, '.git'))
print(('\nWARNING: \'%s\' has been moved from DEPS to a higher '
'level checkout. The git folder containing all the local'
' branches has been saved to %s.\n'
'If you don\'t care about its state you can safely '
'remove that folder to free up space.') %
(entry, save_dir))
continue
if scm_root in full_entries:
logging.info('%s is part of a higher level checkout, not '
'removing.', scm.GetCheckoutRoot())
logging.info('%s is part of a higher level checkout, not removing',
scm.GetCheckoutRoot())
continue
file_list = []

20
gclient_scm.py
View File

@@ -389,6 +389,20 @@ class GitWrapper(SCMWrapper):
if mirror:
url = mirror.mirror_path
# If we are going to introduce a new project, there is a possibility that
# we are syncing back to a state where the project was originally a
# sub-project rolled by DEPS (realistic case: crossing the Blink merge point
# syncing backwards, when Blink was a DEPS entry and not part of src.git).
# In such case, we might have a backup of the former .git folder, which can
# be used to avoid re-fetching the entire repo again (useful for bisects).
backup_dir = self.GetGitBackupDirPath()
target_dir = os.path.join(self.checkout_path, '.git')
if os.path.exists(backup_dir) and not os.path.exists(target_dir):
gclient_utils.safe_makedirs(self.checkout_path)
os.rename(backup_dir, target_dir)
# Reset to a clean state
self._Run(['reset', '--hard', 'HEAD'], options)
if (not os.path.exists(self.checkout_path) or
(os.path.isdir(self.checkout_path) and
not os.path.exists(os.path.join(self.checkout_path, '.git')))):
@@ -799,6 +813,12 @@ class GitWrapper(SCMWrapper):
base_url = self.url
return base_url[:base_url.rfind('/')] + url
def GetGitBackupDirPath(self):
"""Returns the path where the .git folder for the current project can be
staged/restored. Use case: subproject moved from DEPS <-> outer project."""
return os.path.join(self._root_dir,
'old_' + self.relpath.replace(os.sep, '_')) + '.git'
def _GetMirror(self, url, options):
"""Get a git_cache.Mirror object for the argument url."""
if not git_cache.Mirror.GetCachePath():

10
scm.py
View File

@@ -442,6 +442,16 @@ class GIT(object):
except (OSError, subprocess2.CalledProcessError):
return False
@staticmethod
def IsDirectoryVersioned(cwd, relative_dir):
"""Checks whether the given |relative_dir| is part of cwd's repo."""
return bool(GIT.Capture(['ls-tree', 'HEAD', relative_dir], cwd=cwd))
@staticmethod
def CleanupDir(cwd, relative_dir):
"""Cleans up untracked file inside |relative_dir|."""
return bool(GIT.Capture(['clean', '-df', relative_dir], cwd=cwd))
@staticmethod
def GetGitSvnHeadRev(cwd):
"""Gets the most recently pulled git-svn revision."""

34
testing_support/fake_repos.py
View File

@@ -828,6 +828,40 @@ class FakeRepoSkiaDEPS(FakeReposBase):
})
class FakeRepoBlinkDEPS(FakeReposBase):
"""Simulates the Blink DEPS transition in Chrome."""
NB_GIT_REPOS = 2
DEPS_pre = 'deps = {"src/third_party/WebKit": "%(git_base)srepo_2",}'
DEPS_post = 'deps = {}'
def populateGit(self):
# Blink repo.
self._commit_git('repo_2', {
'OWNERS': 'OWNERS-pre',
'Source/exists_always': '_ignored_',
'Source/exists_before_but_not_after': '_ignored_',
})
# Chrome repo.
self._commit_git('repo_1', {
'DEPS': self.DEPS_pre % {'git_base': self.git_base},
'myfile': 'myfile@1',
'.gitignore': '/third_party/WebKit',
})
self._commit_git('repo_1', {
'DEPS': self.DEPS_post % {'git_base': self.git_base},
'myfile': 'myfile@2',
'.gitignore': '',
'third_party/WebKit/OWNERS': 'OWNERS-post',
'third_party/WebKit/Source/exists_always': '_ignored_',
'third_party/WebKit/Source/exists_after_but_not_before': '_ignored',
})
def populateSvn(self):
raise NotImplementedError()
class FakeReposTestBase(trial_dir.TestCase):
"""This is vaguely inspired by twisted."""
# Static FakeRepos instances. Lazy loaded.

4
tests/gclient_scm_test.py
View File

@@ -1245,6 +1245,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
self.root_dir = '/tmp' if sys.platform != 'win32' else 't:\\tmp'
self.relpath = 'fake'
self.base_path = os.path.join(self.root_dir, self.relpath)
self.backup_base_path = os.path.join(self.root_dir,
'old_%s.git' % self.relpath)
def tearDown(self):
BaseTestCase.tearDown(self)
@@ -1354,6 +1356,7 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
gclient_scm.os.path.isdir(
os.path.join(self.base_path, '.git', 'hooks')).AndReturn(False)
gclient_scm.os.path.exists(self.backup_base_path).AndReturn(False)
gclient_scm.os.path.exists(self.base_path).AndReturn(True)
gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
gclient_scm.os.path.exists(os.path.join(self.base_path, '.git')
@@ -1384,6 +1387,7 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
gclient_scm.os.path.isdir(
os.path.join(self.base_path, '.git', 'hooks')).AndReturn(False)
gclient_scm.os.path.exists(self.backup_base_path).AndReturn(False)
gclient_scm.os.path.exists(self.base_path).AndReturn(True)
gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
gclient_scm.os.path.exists(os.path.join(self.base_path, '.git')

134
tests/gclient_smoketest.py
View File

@@ -22,7 +22,7 @@ sys.path.insert(0, ROOT_DIR)
from testing_support.fake_repos import join, write
from testing_support.fake_repos import FakeReposTestBase, FakeRepoTransitive, \
FakeRepoSkiaDEPS
FakeRepoSkiaDEPS, FakeRepoBlinkDEPS
import gclient_utils
import scm as gclient_scm
@@ -1538,6 +1538,138 @@ class SkiaDEPSTransitionSmokeTest(GClientSmokeBase):
skia_src), src_git_url)
class BlinkDEPSTransitionSmokeTest(GClientSmokeBase):
"""Simulate the behavior of bisect bots as they transition across the Blink
DEPS change."""
FAKE_REPOS_CLASS = FakeRepoBlinkDEPS
def setUp(self):
super(BlinkDEPSTransitionSmokeTest, self).setUp()
self.enabled = self.FAKE_REPOS.set_up_git()
self.checkout_path = os.path.join(self.root_dir, 'src')
self.blink = os.path.join(self.checkout_path, 'third_party', 'WebKit')
self.blink_git_url = self.FAKE_REPOS.git_base + 'repo_2'
self.pre_merge_sha = self.githash('repo_1', 1)
self.post_merge_sha = self.githash('repo_1', 2)
def CheckStatusPreMergePoint(self):
self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
self.blink), self.blink_git_url)
self.assertTrue(os.path.exists(join(self.blink, '.git')))
self.assertTrue(os.path.exists(join(self.blink, 'OWNERS')))
with open(join(self.blink, 'OWNERS')) as f:
owners_content = f.read()
self.assertEqual('OWNERS-pre', owners_content, 'OWNERS not updated')
self.assertTrue(os.path.exists(join(self.blink, 'Source', 'exists_always')))
self.assertTrue(os.path.exists(
join(self.blink, 'Source', 'exists_before_but_not_after')))
self.assertFalse(os.path.exists(
join(self.blink, 'Source', 'exists_after_but_not_before')))
def CheckStatusPostMergePoint(self):
# Check that the contents still exists
self.assertTrue(os.path.exists(join(self.blink, 'OWNERS')))
with open(join(self.blink, 'OWNERS')) as f:
owners_content = f.read()
self.assertEqual('OWNERS-post', owners_content, 'OWNERS not updated')
self.assertTrue(os.path.exists(join(self.blink, 'Source', 'exists_always')))
# Check that file removed between the branch point are actually deleted.
self.assertTrue(os.path.exists(
join(self.blink, 'Source', 'exists_after_but_not_before')))
self.assertFalse(os.path.exists(
join(self.blink, 'Source', 'exists_before_but_not_after')))
# But not the .git folder
self.assertFalse(os.path.exists(join(self.blink, '.git')))
def testBlinkDEPSChangeUsingGclient(self):
"""Checks that {src,blink} repos are consistent when syncing going back and
forth using gclient sync src@revision."""
if not self.enabled:
return
self.gclient(['config', '--spec',
'solutions=['
'{"name": "src",'
' "url": "' + self.git_base + 'repo_1",'
'}]'])
# Go back and forth two times.
for _ in xrange(2):
res = self.gclient(['sync', '--jobs', '1',
'--revision', 'src@%s' % self.pre_merge_sha])
self.assertEqual(res[2], 0, 'DEPS change sync failed.')
self.CheckStatusPreMergePoint()
res = self.gclient(['sync', '--jobs', '1',
'--revision', 'src@%s' % self.post_merge_sha])
self.assertEqual(res[2], 0, 'DEPS change sync failed.')
self.CheckStatusPostMergePoint()
def testBlinkDEPSChangeUsingGit(self):
"""Like testBlinkDEPSChangeUsingGclient, but move the main project using
directly git and not gclient sync."""
if not self.enabled:
return
self.gclient(['config', '--spec',
'solutions=['
'{"name": "src",'
' "url": "' + self.git_base + 'repo_1",'
' "managed": False,'
'}]'])
# Perform an initial sync to bootstrap the repo.
res = self.gclient(['sync', '--jobs', '1'])
self.assertEqual(res[2], 0, 'Initial gclient sync failed.')
# Go back and forth two times.
for _ in xrange(2):
subprocess2.check_call(['git', 'checkout', '-q', self.pre_merge_sha],
cwd=self.checkout_path)
res = self.gclient(['sync', '--jobs', '1'])
self.assertEqual(res[2], 0, 'gclient sync failed.')
self.CheckStatusPreMergePoint()
subprocess2.check_call(['git', 'checkout', '-q', self.post_merge_sha],
cwd=self.checkout_path)
res = self.gclient(['sync', '--jobs', '1'])
self.assertEqual(res[2], 0, 'DEPS change sync failed.')
self.CheckStatusPostMergePoint()
def testBlinkLocalBranchesArePreserved(self):
"""Checks that the state of local git branches are effectively preserved
when going back and forth."""
if not self.enabled:
return
self.gclient(['config', '--spec',
'solutions=['
'{"name": "src",'
' "url": "' + self.git_base + 'repo_1",'
'}]'])
# Initialize to pre-merge point.
self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha])
self.CheckStatusPreMergePoint()
# Create a branch named "foo".
subprocess2.check_call(['git', 'checkout', '-qB', 'foo'],
cwd=self.blink)
# Cross the pre-merge point.
self.gclient(['sync', '--revision', 'src@%s' % self.post_merge_sha])
self.CheckStatusPostMergePoint()
# Go backwards and check that we still have the foo branch.
self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha])
self.CheckStatusPreMergePoint()
subprocess2.check_call(
['git', 'show-ref', '-q', '--verify', 'refs/heads/foo'], cwd=self.blink)
class GClientSmokeFromCheckout(GClientSmokeBase):
# WebKit abuses this. It has a .gclient and a DEPS from a checkout.
def setUp(self):

2
tests/scm_unittest.py
View File

@@ -77,6 +77,7 @@ class GitWrapperTestCase(BaseSCMTestCase):
'AssertVersion',
'Capture',
'CaptureStatus',
'CleanupDir',
'current_version',
'FetchUpstreamTuple',
'GenerateDiff',
@@ -92,6 +93,7 @@ class GitWrapperTestCase(BaseSCMTestCase):
'GetSha1ForSvnRev',
'GetSVNBranch',
'GetUpstreamBranch',
'IsDirectoryVersioned',
'IsGitSvn',
'IsInsideWorkTree',
'IsValidRevision',