Tools/git-repo
1
0
Fork 0
Code Activity

init: added --use-superproject option to clone superproject.

Browse source 
Added --no-use-superproject to repo and init.py to disable use of
manifest superprojects.

Replaced the term "sha" with "commit id".

Added _GetBranch method to Superproject object.

Moved shared code between init and sync into SyncSuperproject function.
This function either does git clone or git fetch. If git fetch fails
it does git clone.

Changed Superproject constructor to accept manifest, repodir and branch
to avoid passing them to multiple functions as argument.

Changed functions that were raising exceptions to return either True
or False.

Saved the --use-superproject option in config as repo.superproject.
Updated internal-fs-layout.md document.

Updated the tests to work with the new API changes in Superproject.

Performance for the first time sync has improved from 20 minutes to
around 15 minutes.

Tested the code with the following commands.

$ ./run_tests -v

Tested the sync code by using repo_dev alias and pointing to this CL.

$ repo init took around 20 seconds longer because of cloning of superproject.

$ time repo_dev init -u sso://android.git.corp.google.com/platform/manifest -b master --partial-clone --clone-filter=blob:limit=10M --repo-rev=main --use-superproject
...
real	0m35.919s
user	0m21.947s
sys	0m8.977s

First run
$ time repo sync --use-superproject
...
real	16m41.982s
user	100m6.916s
sys	19m18.753s

No difference in repo sync time after the first run.

Bug: [google internal] b/179090734
Bug: https://crbug.com/gerrit/13709
Bug: https://crbug.com/gerrit/13707

Change-Id: I12df92112f46e001dfbc6f12cd633c3a15cf924b
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/296382
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Raman Tenneti <rtenneti@google.com>
This commit is contained in:
Raman Tenneti 2021-02-09 00:26:31 -08:00
parent e3315bb49a
commit 21dce3d8b3
6 changed files with 183 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

154
git_superproject.py
View file

@ -12,21 +12,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provide functionality to get all projects and their SHAs from Superproject.
"""Provide functionality to get all projects and their commit ids from Superproject.
For more information on superproject, check out:
https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
Examples:
superproject = Superproject()
project_shas = superproject.GetAllProjectsSHAs()
project_commit_ids = superproject.UpdateProjectsRevisionId(projects)
"""
import os
import sys
from error import BUG_REPORT_URL, GitError
from error import BUG_REPORT_URL
from git_command import GitCommand
from git_refs import R_HEADS
import platform_utils
_SUPERPROJECT_GIT_NAME = 'superproject.git'
@ -34,19 +35,24 @@ _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
class Superproject(object):
"""Get SHAs from superproject.
"""Get commit ids from superproject.
It does a 'git clone' of superproject and 'git ls-tree' to get list of SHAs for all projects.
It contains project_shas which is a dictionary with project/sha entries.
It does a 'git clone' of superproject and 'git ls-tree' to get list of commit ids
for all projects. It contains project_commit_ids which is a dictionary with
project/commit id entries.
"""
def __init__(self, repodir, superproject_dir='exp-superproject'):
def __init__(self, manifest, repodir, superproject_dir='exp-superproject'):
"""Initializes superproject.
Args:
manifest: A Manifest object that is to be written to a file.
repodir: Path to the .repo/ dir for holding all internal checkout state.
It must be in the top directory of the repo client checkout.
superproject_dir: Relative path under |repodir| to checkout superproject.
"""
self._project_shas = None
self._project_commit_ids = None
self._manifest = manifest
self._branch = self._GetBranch()
self._repodir = os.path.abspath(repodir)
self._superproject_dir = superproject_dir
self._superproject_path = os.path.join(self._repodir, superproject_dir)
@ -56,25 +62,35 @@ class Superproject(object):
_SUPERPROJECT_GIT_NAME)
@property
def project_shas(self):
"""Returns a dictionary of projects and their SHAs."""
return self._project_shas
def project_commit_ids(self):
"""Returns a dictionary of projects and their commit ids."""
return self._project_commit_ids
def _Clone(self, url, branch=None):
"""Do a 'git clone' for the given url and branch.
def _GetBranch(self):
"""Returns the branch name for getting the approved manifest."""
p = self._manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
if not b:
return None
branch = b.merge
if branch and branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
return branch
def _Clone(self, url):
"""Do a 'git clone' for the given url.
Args:
url: superproject's url to be passed to git clone.
branch: The branchname to be passed as argument to git clone.
Returns:
True if 'git clone <url> <branch>' is successful, or False.
True if git clone is successful, or False.
"""
if not os.path.exists(self._superproject_path):
os.mkdir(self._superproject_path)
cmd = ['clone', url, '--filter', 'blob:none', '--bare']
if branch:
cmd += ['--branch', branch]
if self._branch:
cmd += ['--branch', self._branch]
p = GitCommand(None,
cmd,
cwd=self._superproject_path,
@ -112,22 +128,20 @@ class Superproject(object):
return False
return True
def _LsTree(self, branch='HEAD'):
"""Returns the data from 'git ls-tree -r <branch>'.
def _LsTree(self):
"""Returns the data from 'git ls-tree ...'.
Works only in git repositories.
Args:
branch: The branchname to be passed as argument to git ls-tree.
Returns:
data: data returned from 'git ls-tree -r HEAD' instead of None.
data: data returned from 'git ls-tree ...' instead of None.
"""
if not os.path.exists(self._work_git):
print('git ls-tree missing drectory: %s' % self._work_git,
file=sys.stderr)
return None
data = None
branch = 'HEAD' if not self._branch else self._branch
cmd = ['ls-tree', '-z', '-r', branch]
p = GitCommand(None,
@ -145,18 +159,25 @@ class Superproject(object):
retval, p.stderr), file=sys.stderr)
return data
def _GetAllProjectsSHAs(self, url, branch=None):
"""Get SHAs for all projects from superproject and save them in _project_shas.
Args:
url: superproject's url to be passed to git clone or fetch.
branch: The branchname to be passed as argument to git clone or fetch.
def Sync(self):
"""Sync superproject either by git clone/fetch.
Returns:
A dictionary with the projects/SHAs instead of None.
True if sync of superproject is successful, or False.
"""
print('WARNING: --use-superproject is experimental and not '
'for general use', file=sys.stderr)
if not self._manifest.superproject:
print('error: superproject tag is not defined in manifest',
file=sys.stderr)
return False
url = self._manifest.superproject['remote'].url
if not url:
raise ValueError('url argument is not supplied.')
print('error: superproject URL is not defined in manifest',
file=sys.stderr)
return False
do_clone = True
if os.path.exists(self._superproject_path):
@ -166,35 +187,44 @@ class Superproject(object):
else:
do_clone = False
if do_clone:
if not self._Clone(url, branch):
raise GitError('git clone failed for url: %s' % url)
if not self._Clone(url):
print('error: git clone failed for url: %s' % url, file=sys.stderr)
return False
return True
data = self._LsTree(branch)
def _GetAllProjectsCommitIds(self):
"""Get commit ids for all projects from superproject and save them in _project_commit_ids.
Returns:
A dictionary with the projects/commit ids on success, otherwise None.
"""
if not self.Sync():
return None
data = self._LsTree()
if not data:
raise GitError('git ls-tree failed for url: %s' % url)
print('error: git ls-tree failed for superproject', file=sys.stderr)
return None
# Parse lines like the following to select lines starting with '160000' and
# build a dictionary with project path (last element) and its SHA (3rd element).
# build a dictionary with project path (last element) and its commit id (3rd element).
#
# 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00
# 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00
shas = {}
commit_ids = {}
for line in data.split('\x00'):
ls_data = line.split(None, 3)
if not ls_data:
break
if ls_data[0] == '160000':
shas[ls_data[3]] = ls_data[2]
commit_ids[ls_data[3]] = ls_data[2]
self._project_shas = shas
return shas
self._project_commit_ids = commit_ids
return commit_ids
def _WriteManfiestFile(self, manifest):
def _WriteManfiestFile(self):
"""Writes manifest to a file.
Args:
manifest: A Manifest object that is to be written to a file.
Returns:
manifest_path: Path name of the file into which manifest is written instead of None.
"""
@ -203,7 +233,7 @@ class Superproject(object):
self._superproject_path,
file=sys.stderr)
return None
manifest_str = manifest.ToXml().toxml()
manifest_str = self._manifest.ToXml().toxml()
manifest_path = self._manifest_path
try:
with open(manifest_path, 'w', encoding='utf-8') as fp:
@ -215,40 +245,34 @@ class Superproject(object):
return None
return manifest_path
def UpdateProjectsRevisionId(self, manifest, projects, url, branch=None):
"""Update revisionId of every project in projects with the SHA.
def UpdateProjectsRevisionId(self, projects):
"""Update revisionId of every project in projects with the commit id.
Args:
manifest: A Manifest object that is to be written to a file.
projects: List of projects whose revisionId needs to be updated.
url: superproject's url to be passed to git clone or fetch.
branch: The branchname to be passed as argument to git clone or fetch.
Returns:
manifest_path: Path name of the overriding manfiest file instead of None.
"""
try:
shas = self._GetAllProjectsSHAs(url=url, branch=branch)
except Exception as e:
print('error: Cannot get project SHAs for %s: %s: %s' %
(url, type(e).__name__, str(e)),
file=sys.stderr)
commit_ids = self._GetAllProjectsCommitIds()
if not commit_ids:
print('error: Cannot get project commit ids from manifest', file=sys.stderr)
return None
projects_missing_shas = []
projects_missing_commit_ids = []
for project in projects:
path = project.relpath
if not path:
continue
sha = shas.get(path)
if sha:
project.SetRevisionId(sha)
commit_id = commit_ids.get(path)
if commit_id:
project.SetRevisionId(commit_id)
else:
projects_missing_shas.append(path)
if projects_missing_shas:
print('error: please file a bug using %s to report missing shas for: %s' %
(BUG_REPORT_URL, projects_missing_shas), file=sys.stderr)
projects_missing_commit_ids.append(path)
if projects_missing_commit_ids:
print('error: please file a bug using %s to report missing commit_ids for: %s' %
(BUG_REPORT_URL, projects_missing_commit_ids), file=sys.stderr)
return None
manifest_path = self._WriteManfiestFile(manifest)
manifest_path = self._WriteManfiestFile()
return manifest_path