From e284ad1d1a2c6fa0e0ac800e87b2607f9bda339e Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 4 Nov 2008 07:37:10 -0800 Subject: [PATCH] Add 'repo init --mirror' to download a complete forrest The mirror option downloads a complete forrest (as described by the manifest) and creates a replica of the remote repositories rather than a client working directory. This permits other clients to sync off the mirror site. A mirror can be positioned in a "DMZ", where the mirror executes "repo sync" to obtain changes from the external upstream and clients inside the protected zone operate off the mirror only, and therefore do not require direct git:// access to the external upstream repositories. Signed-off-by: Shawn O. Pearce --- git_config.py | 10 +++++---- manifest.py | 57 +++++++++++++++++++++++++++++++++++++++++++++---- project.py | 36 +++++++++++++++++++++++++------ repo | 5 ++++- subcmds/init.py | 16 ++++++++++++-- subcmds/sync.py | 5 +++-- 6 files changed, 109 insertions(+), 20 deletions(-) diff --git a/git_config.py b/git_config.py index 76031a0e2..9d5162e76 100644 --- a/git_config.py +++ b/git_config.py @@ -285,12 +285,14 @@ class Remote(object): return True return False - def ResetFetch(self): + def ResetFetch(self, mirror=False): """Set the fetch refspec to its default value. """ - self.fetch = [RefSpec(True, - 'refs/heads/*', - 'refs/remotes/%s/*' % self.name)] + if mirror: + dst = 'refs/heads/*' + else: + dst = 'refs/remotes/%s/*' % self.name + self.fetch = [RefSpec(True, 'refs/heads/*', dst)] def Save(self): """Save this remote to the configuration. diff --git a/manifest.py b/manifest.py index b928cdfee..ea68b6824 100644 --- a/manifest.py +++ b/manifest.py @@ -88,6 +88,10 @@ class Manifest(object): self._Load() return self._default + @property + def IsMirror(self): + return self.manifestProject.config.GetBoolean('repo.mirror') + def _Unload(self): self._loaded = False self._projects = {} @@ -114,6 +118,10 @@ class Manifest(object): finally: self.manifestFile = real + if self.IsMirror: + self._AddMetaProjectMirror(self.repoProject) + self._AddMetaProjectMirror(self.manifestProject) + self._loaded = True def _ParseManifest(self, is_root_file): @@ -157,6 +165,40 @@ class Manifest(object): (project.name, self.manifestFile) self._projects[project.name] = project + def _AddMetaProjectMirror(self, m): + name = None + m_url = m.GetRemote(m.remote.name).url + if m_url.endswith('/.git'): + raise ManifestParseError, 'refusing to mirror %s' % m_url + + if self._default and self._default.remote: + url = self._default.remote.fetchUrl + if not url.endswith('/'): + url += '/' + if m_url.startswith(url): + remote = self._default.remote + name = m_url[len(url):] + + if name is None: + s = m_url.rindex('/') + 1 + remote = Remote('origin', fetch = m_url[:s]) + name = m_url[s:] + + if name.endswith('.git'): + name = name[:-4] + + if name not in self._projects: + m.PreSync() + gitdir = os.path.join(self.topdir, '%s.git' % name) + project = Project(manifest = self, + name = name, + remote = remote, + gitdir = gitdir, + worktree = None, + relpath = None, + revision = m.revision) + self._projects[project.name] = project + def _ParseRemote(self, node): """ reads a element from the manifest file @@ -214,8 +256,13 @@ class Manifest(object): "project %s path cannot be absolute in %s" % \ (name, self.manifestFile) - worktree = os.path.join(self.topdir, path) - gitdir = os.path.join(self.repodir, 'projects/%s.git' % path) + if self.IsMirror: + relpath = None + worktree = None + gitdir = os.path.join(self.topdir, '%s.git' % name) + else: + worktree = os.path.join(self.topdir, path) + gitdir = os.path.join(self.repodir, 'projects/%s.git' % path) project = Project(manifest = self, name = name, @@ -242,8 +289,10 @@ class Manifest(object): def _ParseCopyFile(self, project, node): src = self._reqatt(node, 'src') dest = self._reqatt(node, 'dest') - # src is project relative, and dest is relative to the top of the tree - project.AddCopyFile(src, os.path.join(self.topdir, dest)) + if not self.IsMirror: + # src is project relative; + # dest is relative to the top of the tree + project.AddCopyFile(src, os.path.join(self.topdir, dest)) def _get_remote(self, node): name = node.getAttribute('remote') diff --git a/project.py b/project.py index 0637f4bf4..1cfaaae5e 100644 --- a/project.py +++ b/project.py @@ -211,7 +211,10 @@ class Project(object): gitdir = self.gitdir, defaults = self.manifest.globalConfig) - self.work_git = self._GitGetByExec(self, bare=False) + if self.worktree: + self.work_git = self._GitGetByExec(self, bare=False) + else: + self.work_git = None self.bare_git = self._GitGetByExec(self, bare=True) @property @@ -489,14 +492,23 @@ class Project(object): print >>sys.stderr print >>sys.stderr, 'Initializing project %s ...' % self.name self._InitGitDir() + self._InitRemote() for r in self.extraRemotes.values(): if not self._RemoteFetch(r.name): return False if not self._RemoteFetch(): return False - self._RepairAndroidImportErrors() - self._InitMRef() + + if self.worktree: + self._RepairAndroidImportErrors() + self._InitMRef() + else: + self._InitMirrorHead() + try: + os.remove(os.path.join(self.gitdir, 'FETCH_HEAD')) + except OSError: + pass return True def PostRepoUpgrade(self): @@ -792,9 +804,11 @@ class Project(object): def _RemoteFetch(self, name=None): if not name: name = self.remote.name - return GitCommand(self, - ['fetch', name], - bare = True).Wait() == 0 + cmd = ['fetch'] + if not self.worktree: + cmd.append('--update-head-ok') + cmd.append(name) + return GitCommand(self, cmd, bare = True).Wait() == 0 def _Checkout(self, rev, quiet=False): cmd = ['checkout'] @@ -874,7 +888,10 @@ class Project(object): remote.url = url remote.review = self.remote.reviewUrl - remote.ResetFetch() + if self.worktree: + remote.ResetFetch(mirror=False) + else: + remote.ResetFetch(mirror=True) remote.Save() for r in self.extraRemotes.values(): @@ -897,6 +914,11 @@ class Project(object): dst = remote.ToLocal(self.revision) self.bare_git.symbolic_ref('-m', msg, ref, dst) + def _InitMirrorHead(self): + dst = self.GetRemote(self.remote.name).ToLocal(self.revision) + msg = 'manifest set to %s' % self.revision + self.bare_git.SetHead(dst, message=msg) + def _InitWorkTree(self): dotgit = os.path.join(self.worktree, '.git') if not os.path.exists(dotgit): diff --git a/repo b/repo index 9f107a932..bfa4ca3c0 100755 --- a/repo +++ b/repo @@ -28,7 +28,7 @@ if __name__ == '__main__': del magic # increment this whenever we make important changes to this script -VERSION = (1, 6) +VERSION = (1, 7) # increment this if the MAINTAINER_KEYS block is modified KEYRING_VERSION = (1,0) @@ -115,6 +115,9 @@ group.add_option('-b', '--manifest-branch', group.add_option('-m', '--manifest-name', dest='manifest_name', help='initial manifest file', metavar='NAME.xml') +group.add_option('--mirror', + dest='mirror', action='store_true', + help='mirror the forrest') # Tool group = init_optparse.add_option_group('Version options') diff --git a/subcmds/init.py b/subcmds/init.py index 03f358d19..ad28a6117 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -57,6 +57,10 @@ default.xml will be used. g.add_option('-m', '--manifest-name', dest='manifest_name', default='default.xml', help='initial manifest file', metavar='NAME.xml') + g.add_option('--mirror', + dest='mirror', action='store_true', + help='mirror the forrest') + # Tool g = p.add_option_group('Version options') @@ -112,6 +116,9 @@ default.xml will be used. r.ResetFetch() r.Save() + if opt.mirror: + m.config.SetString('repo.mirror', 'true') + m.Sync_NetworkHalf() m.Sync_LocalHalf() m.StartBranch('default') @@ -185,9 +192,14 @@ default.xml will be used. self._SyncManifest(opt) self._LinkManifest(opt.manifest_name) - if os.isatty(0) and os.isatty(1): + if os.isatty(0) and os.isatty(1) and not opt.mirror: self._ConfigureUser() self._ConfigureColor() + if opt.mirror: + type = 'mirror ' + else: + type = '' + print '' - print 'repo initialized in %s' % self.manifest.topdir + print 'repo %sinitialized in %s' % (type, self.manifest.topdir) diff --git a/subcmds/sync.py b/subcmds/sync.py index 9af123226..8050e515d 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -102,8 +102,9 @@ the manifest. self._Fetch(*missing) for project in all: - if not project.Sync_LocalHalf(): - sys.exit(1) + if project.worktree: + if not project.Sync_LocalHalf(): + sys.exit(1) def _VerifyTag(project):