1
0
Fork 0

command: add a helper for the parallel execution boilerplate

Now that we have a bunch of subcommands doing parallel execution, a
common pattern arises that we can factor out for most of them.  We
leave forall alone as it's a bit too complicated atm to cut over.

Change-Id: I3617a4f7c66142bcd1ab030cb4cca698a65010ac
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/301942
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Chris Mcdonald <cjmcdonald@google.com>
This commit is contained in:
Mike Frysinger 2021-03-01 00:56:38 -05:00
parent b8bf291ddb
commit b5d075d04f
10 changed files with 145 additions and 143 deletions

View file

@ -13,10 +13,10 @@
# limitations under the License.
import itertools
import multiprocessing
import sys
from color import Coloring
from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
from command import Command, DEFAULT_LOCAL_JOBS
class BranchColoring(Coloring):
@ -102,15 +102,19 @@ is shown, then the branch appears in all projects.
out = BranchColoring(self.manifest.manifestProject.config)
all_branches = {}
project_cnt = len(projects)
with multiprocessing.Pool(processes=opt.jobs) as pool:
project_branches = pool.imap_unordered(
expand_project_to_branches, projects, chunksize=WORKER_BATCH_SIZE)
for name, b in itertools.chain.from_iterable(project_branches):
def _ProcessResults(_pool, _output, results):
for name, b in itertools.chain.from_iterable(results):
if name not in all_branches:
all_branches[name] = BranchInfo(name)
all_branches[name].add(b)
self.ExecuteInParallel(
opt.jobs,
expand_project_to_branches,
projects,
callback=_ProcessResults)
names = sorted(all_branches)
if not names: