/[gentoo-x86]/eclass/multiprocessing.eclass
Gentoo

Contents of /eclass/multiprocessing.eclass

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.8 - (show annotations) (download)
Sat Dec 21 09:40:37 2013 UTC (7 months, 1 week ago) by vapier
Branch: MAIN
Changes since 1.7: +19 -2 lines
add support for bash-3.2 which lacks $BASHPID as pointed out by Ryan Hill

1 # Copyright 1999-2013 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
3 # $Header: /var/cvsroot/gentoo-x86/eclass/multiprocessing.eclass,v 1.7 2013/12/07 09:14:15 vapier Exp $
4
5 # @ECLASS: multiprocessing.eclass
6 # @MAINTAINER:
7 # base-system@gentoo.org
8 # @AUTHOR:
9 # Brian Harring <ferringb@gentoo.org>
10 # Mike Frysinger <vapier@gentoo.org>
11 # @BLURB: parallelization with bash (wtf?)
12 # @DESCRIPTION:
13 # The multiprocessing eclass contains a suite of functions that allow ebuilds
14 # to quickly run things in parallel using shell code.
15 #
16 # It has two modes: pre-fork and post-fork. If you don't want to dive into any
17 # more nuts & bolts, just use the pre-fork mode. For main threads that mostly
18 # spawn children and then wait for them to finish, use the pre-fork mode. For
19 # main threads that do a bit of processing themselves, use the post-fork mode.
20 # You may mix & match them for longer computation loops.
21 # @EXAMPLE:
22 #
23 # @CODE
24 # # First initialize things:
25 # multijob_init
26 #
27 # # Then hash a bunch of files in parallel:
28 # for n in {0..20} ; do
29 # multijob_child_init md5sum data.${n} > data.${n}
30 # done
31 #
32 # # Then wait for all the children to finish:
33 # multijob_finish
34 # @CODE
35
36 if [[ ${___ECLASS_ONCE_MULTIPROCESSING} != "recur -_+^+_- spank" ]] ; then
37 ___ECLASS_ONCE_MULTIPROCESSING="recur -_+^+_- spank"
38
39 # @FUNCTION: bashpid
40 # @DESCRIPTION:
41 # Return the process id of the current sub shell. This is to support bash
42 # versions older than 4.0 that lack $BASHPID support natively. Simply do:
43 # echo ${BASHPID:-$(bashpid)}
44 #
45 # Note: Using this func in any other way than the one above is not supported.
46 bashpid() {
47 # Running bashpid plainly will return incorrect results. This func must
48 # be run in a subshell of the current subshell to get the right pid.
49 # i.e. This will show the wrong value:
50 # bashpid
51 # But this will show the right value:
52 # (bashpid)
53 sh -c 'echo ${PPID}'
54 }
55
56 # @FUNCTION: makeopts_jobs
57 # @USAGE: [${MAKEOPTS}]
58 # @DESCRIPTION:
59 # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
60 # specified therein. Useful for running non-make tools in parallel too.
61 # i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
62 # number as bash normalizes it to [0, 255]. If the flags haven't specified a
63 # -j flag, then "1" is shown as that is the default `make` uses. Since there's
64 # no way to represent infinity, we return 999 if the user has -j without a number.
65 makeopts_jobs() {
66 [[ $# -eq 0 ]] && set -- ${MAKEOPTS}
67 # This assumes the first .* will be more greedy than the second .*
68 # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
69 local jobs=$(echo " $* " | sed -r -n \
70 -e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \
71 -e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p')
72 echo ${jobs:-1}
73 }
74
75 # @FUNCTION: makeopts_loadavg
76 # @USAGE: [${MAKEOPTS}]
77 # @DESCRIPTION:
78 # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the value set
79 # for load-average. For make and ninja based builds this will mean new jobs are
80 # not only limited by the jobs-value, but also by the current load - which might
81 # get excessive due to I/O and not just due to CPU load.
82 # Be aware that the returned number might be a floating-point number. Test
83 # whether your software supports that.
84 makeopts_loadavg() {
85 [[ $# -eq 0 ]] && set -- ${MAKEOPTS}
86 # This assumes the first .* will be more greedy than the second .*
87 # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
88 local lavg=$(echo " $* " | sed -r -n \
89 -e 's:.*[[:space:]](-l|--load-average[=[:space:]])[[:space:]]*([0-9]+|[0-9]+\.[0-9]+)[^0-9.]*:\2:p' \
90 -e 's:.*[[:space:]](-l|--load-average)[[:space:]].*:999:p')
91 echo ${lavg:-1}
92 }
93
94 # @FUNCTION: multijob_init
95 # @USAGE: [${MAKEOPTS}]
96 # @DESCRIPTION:
97 # Setup the environment for executing code in parallel.
98 # You must call this before any other multijob function.
99 multijob_init() {
100 # When something goes wrong, try to wait for all the children so we
101 # don't leave any zombies around.
102 has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait "
103
104 # Setup a pipe for children to write their pids to when they finish.
105 # We have to allocate two fd's because POSIX has undefined behavior
106 # when you open a FIFO for simultaneous read/write. #487056
107 local pipe="${T}/multijob.pipe"
108 mkfifo -m 600 "${pipe}"
109 redirect_alloc_fd mj_write_fd "${pipe}"
110 redirect_alloc_fd mj_read_fd "${pipe}"
111 rm -f "${pipe}"
112
113 # See how many children we can fork based on the user's settings.
114 mj_max_jobs=$(makeopts_jobs "$@")
115 mj_num_jobs=0
116 }
117
118 # @FUNCTION: multijob_child_init
119 # @USAGE: [--pre|--post] [command to run in background]
120 # @DESCRIPTION:
121 # This function has two forms. You can use it to execute a simple command
122 # in the background (and it takes care of everything else), or you must
123 # call this first thing in your forked child process.
124 #
125 # The --pre/--post options allow you to select the child generation mode.
126 #
127 # @CODE
128 # # 1st form: pass the command line as arguments:
129 # multijob_child_init ls /dev
130 # # Or if you want to use pre/post fork modes:
131 # multijob_child_init --pre ls /dev
132 # multijob_child_init --post ls /dev
133 #
134 # # 2nd form: execute multiple stuff in the background (post fork):
135 # (
136 # multijob_child_init
137 # out=`ls`
138 # if echo "${out}" | grep foo ; then
139 # echo "YEAH"
140 # fi
141 # ) &
142 # multijob_post_fork
143 #
144 # # 2nd form: execute multiple stuff in the background (pre fork):
145 # multijob_pre_fork
146 # (
147 # multijob_child_init
148 # out=`ls`
149 # if echo "${out}" | grep foo ; then
150 # echo "YEAH"
151 # fi
152 # ) &
153 # @CODE
154 multijob_child_init() {
155 local mode="pre"
156 case $1 in
157 --pre) mode="pre" ; shift ;;
158 --post) mode="post"; shift ;;
159 esac
160
161 if [[ $# -eq 0 ]] ; then
162 trap 'echo ${BASHPID:-$(bashpid)} $? >&'${mj_write_fd} EXIT
163 trap 'exit 1' INT TERM
164 else
165 local ret
166 [[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; }
167 ( multijob_child_init ; "$@" ) &
168 [[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; }
169 return ${ret}
170 fi
171 }
172
173 # @FUNCTION: _multijob_fork
174 # @INTERNAL
175 # @DESCRIPTION:
176 # Do the actual book keeping.
177 _multijob_fork() {
178 [[ $# -eq 1 ]] || die "incorrect number of arguments"
179
180 local ret=0
181 [[ $1 == "post" ]] && : $(( ++mj_num_jobs ))
182 if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then
183 multijob_finish_one
184 ret=$?
185 fi
186 [[ $1 == "pre" ]] && : $(( ++mj_num_jobs ))
187 return ${ret}
188 }
189
190 # @FUNCTION: multijob_pre_fork
191 # @DESCRIPTION:
192 # You must call this in the parent process before forking a child process.
193 # If the parallel limit has been hit, it will wait for one child to finish
194 # and return its exit status.
195 multijob_pre_fork() { _multijob_fork pre "$@" ; }
196
197 # @FUNCTION: multijob_post_fork
198 # @DESCRIPTION:
199 # You must call this in the parent process after forking a child process.
200 # If the parallel limit has been hit, it will wait for one child to finish
201 # and return its exit status.
202 multijob_post_fork() { _multijob_fork post "$@" ; }
203
204 # @FUNCTION: multijob_finish_one
205 # @DESCRIPTION:
206 # Wait for a single process to exit and return its exit code.
207 multijob_finish_one() {
208 [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
209
210 local pid ret
211 read -r -u ${mj_read_fd} pid ret || die
212 : $(( --mj_num_jobs ))
213 return ${ret}
214 }
215
216 # @FUNCTION: multijob_finish
217 # @DESCRIPTION:
218 # Wait for all pending processes to exit and return the bitwise or
219 # of all their exit codes.
220 multijob_finish() {
221 local ret=0
222 while [[ ${mj_num_jobs} -gt 0 ]] ; do
223 multijob_finish_one
224 : $(( ret |= $? ))
225 done
226 # Let bash clean up its internal child tracking state.
227 wait
228
229 # Do this after reaping all the children.
230 [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
231
232 # No need to hook anymore.
233 EBUILD_DEATH_HOOKS=${EBUILD_DEATH_HOOKS/ wait / }
234
235 return ${ret}
236 }
237
238 # @FUNCTION: redirect_alloc_fd
239 # @USAGE: <var> <file> [redirection]
240 # @DESCRIPTION:
241 # Find a free fd and redirect the specified file via it. Store the new
242 # fd in the specified variable. Useful for the cases where we don't care
243 # about the exact fd #.
244 redirect_alloc_fd() {
245 local var=$1 file=$2 redir=${3:-"<>"}
246
247 # Make sure /dev/fd is sane on Linux hosts. #479656
248 if [[ ! -L /dev/fd && ${CBUILD} == *linux* ]] ; then
249 eerror "You're missing a /dev/fd symlink to /proc/self/fd."
250 eerror "Please fix the symlink and check your boot scripts (udev/etc...)."
251 die "/dev/fd is broken"
252 fi
253
254 if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then
255 # Newer bash provides this functionality.
256 eval "exec {${var}}${redir}'${file}'"
257 else
258 # Need to provide the functionality ourselves.
259 local fd=10
260 while :; do
261 # Make sure the fd isn't open. It could be a char device,
262 # or a symlink (possibly broken) to something else.
263 if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then
264 eval "exec ${fd}${redir}'${file}'" && break
265 fi
266 [[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?'
267 : $(( ++fd ))
268 done
269 : $(( ${var} = fd ))
270 fi
271 }
272
273 fi

  ViewVC Help
Powered by ViewVC 1.1.20