| 1 |
# Copyright 1999-2012 Gentoo Foundation
|
| 2 |
# Distributed under the terms of the GNU General Public License v2
|
| 3 |
# $Header: $
|
| 4 |
|
| 5 |
# @ECLASS: multiprocessing.eclass
|
| 6 |
# @MAINTAINER:
|
| 7 |
# base-system@gentoo.org
|
| 8 |
# @AUTHOR:
|
| 9 |
# Brian Harring <ferringb@gentoo.org>
|
| 10 |
# Mike Frysinger <vapier@gentoo.org>
|
| 11 |
# @BLURB: parallelization with bash (wtf?)
|
| 12 |
# @DESCRIPTION:
|
| 13 |
# The multiprocessing eclass contains a suite of functions that allow ebuilds
|
| 14 |
# to quickly run things in parallel using shell code.
|
| 15 |
#
|
| 16 |
# It has two modes: pre-fork and post-fork. If you don't want to dive into any
|
| 17 |
# more nuts & bolts, just use the pre-fork mode. For main threads that mostly
|
| 18 |
# spawn children and then wait for them to finish, use the pre-fork mode. For
|
| 19 |
# main threads that do a bit of processing themselves, use the post-fork mode.
|
| 20 |
# You may mix & match them for longer computation loops.
|
| 21 |
# @EXAMPLE:
|
| 22 |
#
|
| 23 |
# @CODE
|
| 24 |
# # First initialize things:
|
| 25 |
# multijob_init
|
| 26 |
#
|
| 27 |
# # Then hash a bunch of files in parallel:
|
| 28 |
# for n in {0..20} ; do
|
| 29 |
# multijob_child_init md5sum data.${n} > data.${n}
|
| 30 |
# done
|
| 31 |
#
|
| 32 |
# # Then wait for all the children to finish:
|
| 33 |
# multijob_finish
|
| 34 |
# @CODE
|
| 35 |
|
| 36 |
if [[ ${___ECLASS_ONCE_MULTIPROCESSING} != "recur -_+^+_- spank" ]] ; then
|
| 37 |
___ECLASS_ONCE_MULTIPROCESSING="recur -_+^+_- spank"
|
| 38 |
|
| 39 |
# @FUNCTION: makeopts_jobs
|
| 40 |
# @USAGE: [${MAKEOPTS}]
|
| 41 |
# @DESCRIPTION:
|
| 42 |
# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
|
| 43 |
# specified therein. Useful for running non-make tools in parallel too.
|
| 44 |
# i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
|
| 45 |
# number as bash normalizes it to [0, 255]. If the flags haven't specified a
|
| 46 |
# -j flag, then "1" is shown as that is the default `make` uses. Since there's
|
| 47 |
# no way to represent infinity, we return 999 if the user has -j without a number.
|
| 48 |
makeopts_jobs() {
|
| 49 |
[[ $# -eq 0 ]] && set -- ${MAKEOPTS}
|
| 50 |
# This assumes the first .* will be more greedy than the second .*
|
| 51 |
# since POSIX doesn't specify a non-greedy match (i.e. ".*?").
|
| 52 |
local jobs=$(echo " $* " | sed -r -n \
|
| 53 |
-e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \
|
| 54 |
-e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p')
|
| 55 |
echo ${jobs:-1}
|
| 56 |
}
|
| 57 |
|
| 58 |
# @FUNCTION: multijob_init
|
| 59 |
# @USAGE: [${MAKEOPTS}]
|
| 60 |
# @DESCRIPTION:
|
| 61 |
# Setup the environment for executing code in parallel.
|
| 62 |
# You must call this before any other multijob function.
|
| 63 |
multijob_init() {
|
| 64 |
# When something goes wrong, try to wait for all the children so we
|
| 65 |
# don't leave any zombies around.
|
| 66 |
has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait"
|
| 67 |
|
| 68 |
# Setup a pipe for children to write their pids to when they finish.
|
| 69 |
local pipe="${T}/multijob.pipe"
|
| 70 |
mkfifo "${pipe}"
|
| 71 |
redirect_alloc_fd mj_control_fd "${pipe}"
|
| 72 |
rm -f "${pipe}"
|
| 73 |
|
| 74 |
# See how many children we can fork based on the user's settings.
|
| 75 |
mj_max_jobs=$(makeopts_jobs "$@")
|
| 76 |
mj_num_jobs=0
|
| 77 |
}
|
| 78 |
|
| 79 |
# @FUNCTION: multijob_child_init
|
| 80 |
# @USAGE: [--pre|--post] [command to run in background]
|
| 81 |
# @DESCRIPTION:
|
| 82 |
# This function has two forms. You can use it to execute a simple command
|
| 83 |
# in the background (and it takes care of everything else), or you must
|
| 84 |
# call this first thing in your forked child process.
|
| 85 |
#
|
| 86 |
# The --pre/--post options allow you to select the child generation mode.
|
| 87 |
#
|
| 88 |
# @CODE
|
| 89 |
# # 1st form: pass the command line as arguments:
|
| 90 |
# multijob_child_init ls /dev
|
| 91 |
# # Or if you want to use pre/post fork modes:
|
| 92 |
# multijob_child_init --pre ls /dev
|
| 93 |
# multijob_child_init --post ls /dev
|
| 94 |
#
|
| 95 |
# # 2nd form: execute multiple stuff in the background (post fork):
|
| 96 |
# (
|
| 97 |
# multijob_child_init
|
| 98 |
# out=`ls`
|
| 99 |
# if echo "${out}" | grep foo ; then
|
| 100 |
# echo "YEAH"
|
| 101 |
# fi
|
| 102 |
# ) &
|
| 103 |
# multijob_post_fork
|
| 104 |
#
|
| 105 |
# # 2nd form: execute multiple stuff in the background (pre fork):
|
| 106 |
# multijob_pre_fork
|
| 107 |
# (
|
| 108 |
# multijob_child_init
|
| 109 |
# out=`ls`
|
| 110 |
# if echo "${out}" | grep foo ; then
|
| 111 |
# echo "YEAH"
|
| 112 |
# fi
|
| 113 |
# ) &
|
| 114 |
# @CODE
|
| 115 |
multijob_child_init() {
|
| 116 |
local mode="pre"
|
| 117 |
case $1 in
|
| 118 |
--pre) mode="pre" ; shift ;;
|
| 119 |
--post) mode="post"; shift ;;
|
| 120 |
esac
|
| 121 |
|
| 122 |
if [[ $# -eq 0 ]] ; then
|
| 123 |
trap 'echo ${BASHPID} $? >&'${mj_control_fd} EXIT
|
| 124 |
trap 'exit 1' INT TERM
|
| 125 |
else
|
| 126 |
local ret
|
| 127 |
[[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; }
|
| 128 |
( multijob_child_init ; "$@" ) &
|
| 129 |
[[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; }
|
| 130 |
return ${ret}
|
| 131 |
fi
|
| 132 |
}
|
| 133 |
|
| 134 |
# @FUNCTION: _multijob_fork
|
| 135 |
# @INTERNAL
|
| 136 |
# @DESCRIPTION:
|
| 137 |
# Do the actual book keeping.
|
| 138 |
_multijob_fork() {
|
| 139 |
[[ $# -eq 1 ]] || die "incorrect number of arguments"
|
| 140 |
|
| 141 |
local ret=0
|
| 142 |
[[ $1 == "post" ]] && : $(( ++mj_num_jobs ))
|
| 143 |
if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then
|
| 144 |
multijob_finish_one
|
| 145 |
ret=$?
|
| 146 |
fi
|
| 147 |
[[ $1 == "pre" ]] && : $(( ++mj_num_jobs ))
|
| 148 |
return ${ret}
|
| 149 |
}
|
| 150 |
|
| 151 |
# @FUNCTION: multijob_pre_fork
|
| 152 |
# @DESCRIPTION:
|
| 153 |
# You must call this in the parent process before forking a child process.
|
| 154 |
# If the parallel limit has been hit, it will wait for one child to finish
|
| 155 |
# and return its exit status.
|
| 156 |
multijob_pre_fork() { _multijob_fork pre "$@" ; }
|
| 157 |
|
| 158 |
# @FUNCTION: multijob_post_fork
|
| 159 |
# @DESCRIPTION:
|
| 160 |
# You must call this in the parent process after forking a child process.
|
| 161 |
# If the parallel limit has been hit, it will wait for one child to finish
|
| 162 |
# and return its exit status.
|
| 163 |
multijob_post_fork() { _multijob_fork post "$@" ; }
|
| 164 |
|
| 165 |
# @FUNCTION: multijob_finish_one
|
| 166 |
# @DESCRIPTION:
|
| 167 |
# Wait for a single process to exit and return its exit code.
|
| 168 |
multijob_finish_one() {
|
| 169 |
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
|
| 170 |
|
| 171 |
local pid ret
|
| 172 |
read -r -u ${mj_control_fd} pid ret || die
|
| 173 |
: $(( --mj_num_jobs ))
|
| 174 |
return ${ret}
|
| 175 |
}
|
| 176 |
|
| 177 |
# @FUNCTION: multijob_finish
|
| 178 |
# @DESCRIPTION:
|
| 179 |
# Wait for all pending processes to exit and return the bitwise or
|
| 180 |
# of all their exit codes.
|
| 181 |
multijob_finish() {
|
| 182 |
local ret=0
|
| 183 |
while [[ ${mj_num_jobs} -gt 0 ]] ; do
|
| 184 |
multijob_finish_one
|
| 185 |
: $(( ret |= $? ))
|
| 186 |
done
|
| 187 |
# Let bash clean up its internal child tracking state.
|
| 188 |
wait
|
| 189 |
|
| 190 |
# Do this after reaping all the children.
|
| 191 |
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
|
| 192 |
|
| 193 |
return ${ret}
|
| 194 |
}
|
| 195 |
|
| 196 |
# @FUNCTION: redirect_alloc_fd
|
| 197 |
# @USAGE: <var> <file> [redirection]
|
| 198 |
# @DESCRIPTION:
|
| 199 |
# Find a free fd and redirect the specified file via it. Store the new
|
| 200 |
# fd in the specified variable. Useful for the cases where we don't care
|
| 201 |
# about the exact fd #.
|
| 202 |
redirect_alloc_fd() {
|
| 203 |
local var=$1 file=$2 redir=${3:-"<>"}
|
| 204 |
|
| 205 |
if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then
|
| 206 |
# Newer bash provides this functionality.
|
| 207 |
eval "exec {${var}}${redir}'${file}'"
|
| 208 |
else
|
| 209 |
# Need to provide the functionality ourselves.
|
| 210 |
local fd=10
|
| 211 |
while :; do
|
| 212 |
# Make sure the fd isn't open. It could be a char device,
|
| 213 |
# or a symlink (possibly broken) to something else.
|
| 214 |
if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then
|
| 215 |
eval "exec ${fd}${redir}'${file}'" && break
|
| 216 |
fi
|
| 217 |
[[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?'
|
| 218 |
: $(( ++fd ))
|
| 219 |
done
|
| 220 |
: $(( ${var} = fd ))
|
| 221 |
fi
|
| 222 |
}
|
| 223 |
|
| 224 |
fi
|
Parent Directory
| 
Revision Log