command: add a helper for the parallel execution boilerplate
Now that we have a bunch of subcommands doing parallel execution, a common pattern arises that we can factor out for most of them. We leave forall alone as it's a bit too complicated atm to cut over. Change-Id: I3617a4f7c66142bcd1ab030cb4cca698a65010ac Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/301942 Tested-by: Mike Frysinger <vapier@google.com> Reviewed-by: Chris Mcdonald <cjmcdonald@google.com>
This commit is contained in:
parent
b8bf291ddb
commit
b5d075d04f
10 changed files with 145 additions and 143 deletions
|
@ -13,10 +13,10 @@
|
|||
# limitations under the License.
|
||||
|
||||
import itertools
|
||||
import multiprocessing
|
||||
import sys
|
||||
|
||||
from color import Coloring
|
||||
from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
|
||||
from command import Command, DEFAULT_LOCAL_JOBS
|
||||
|
||||
|
||||
class BranchColoring(Coloring):
|
||||
|
@ -102,15 +102,19 @@ is shown, then the branch appears in all projects.
|
|||
out = BranchColoring(self.manifest.manifestProject.config)
|
||||
all_branches = {}
|
||||
project_cnt = len(projects)
|
||||
with multiprocessing.Pool(processes=opt.jobs) as pool:
|
||||
project_branches = pool.imap_unordered(
|
||||
expand_project_to_branches, projects, chunksize=WORKER_BATCH_SIZE)
|
||||
|
||||
for name, b in itertools.chain.from_iterable(project_branches):
|
||||
def _ProcessResults(_pool, _output, results):
|
||||
for name, b in itertools.chain.from_iterable(results):
|
||||
if name not in all_branches:
|
||||
all_branches[name] = BranchInfo(name)
|
||||
all_branches[name].add(b)
|
||||
|
||||
self.ExecuteInParallel(
|
||||
opt.jobs,
|
||||
expand_project_to_branches,
|
||||
projects,
|
||||
callback=_ProcessResults)
|
||||
|
||||
names = sorted(all_branches)
|
||||
|
||||
if not names:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue