Tools/git-repo
1
0
Fork 0
Code Activity

command: add a helper for the parallel execution boilerplate

Browse source 
Now that we have a bunch of subcommands doing parallel execution, a
common pattern arises that we can factor out for most of them.  We
leave forall alone as it's a bit too complicated atm to cut over.

Change-Id: I3617a4f7c66142bcd1ab030cb4cca698a65010ac
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/301942
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Chris Mcdonald <cjmcdonald@google.com>
This commit is contained in:
Mike Frysinger 2021-03-01 00:56:38 -05:00
parent b8bf291ddb
commit b5d075d04f
10 changed files with 145 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

24
subcmds/grep.py
View file

@ -13,11 +13,10 @@
# limitations under the License.
import functools
import multiprocessing
import sys
from color import Coloring
from command import DEFAULT_LOCAL_JOBS, PagedCommand, WORKER_BATCH_SIZE
from command import DEFAULT_LOCAL_JOBS, PagedCommand
from error import GitError
from git_command import GitCommand
@ -173,7 +172,7 @@ contain a line that matches both expressions:
return (project, p.Wait(), p.stdout, p.stderr)
@staticmethod
def _ProcessResults(out, full_name, have_rev, results):
def _ProcessResults(full_name, have_rev, _pool, out, results):
git_failed = False
bad_rev = False
have_match = False
@ -256,18 +255,13 @@ contain a line that matches both expressions:
cmd_argv.extend(opt.revision)
cmd_argv.append('--')
process_results = functools.partial(
self._ProcessResults, out, full_name, have_rev)
# NB: Multiprocessing is heavy, so don't spin it up for one job.
if len(projects) == 1 or opt.jobs == 1:
git_failed, bad_rev, have_match = process_results(
self._ExecuteOne(cmd_argv, x) for x in projects)
else:
with multiprocessing.Pool(opt.jobs) as pool:
results = pool.imap(
functools.partial(self._ExecuteOne, cmd_argv), projects,
chunksize=WORKER_BATCH_SIZE)
git_failed, bad_rev, have_match = process_results(results)
git_failed, bad_rev, have_match = self.ExecuteInParallel(
opt.jobs,
functools.partial(self._ExecuteOne, cmd_argv),
projects,
callback=functools.partial(self._ProcessResults, full_name, have_rev),
output=out,
ordered=True)
if git_failed:
sys.exit(1)