1 |
ulm |
1.9 |
# Copyright 1999-2014 Gentoo Foundation |
2 |
vapier |
1.1 |
# Distributed under the terms of the GNU General Public License v2 |
3 |
ulm |
1.9 |
# $Header: /var/cvsroot/gentoo-x86/eclass/multiprocessing.eclass,v 1.8 2013/12/21 09:40:37 vapier Exp $ |
4 |
vapier |
1.1 |
|
5 |
|
|
# @ECLASS: multiprocessing.eclass |
6 |
|
|
# @MAINTAINER: |
7 |
|
|
# base-system@gentoo.org |
8 |
|
|
# @AUTHOR: |
9 |
|
|
# Brian Harring <ferringb@gentoo.org> |
10 |
|
|
# Mike Frysinger <vapier@gentoo.org> |
11 |
|
|
# @BLURB: parallelization with bash (wtf?) |
12 |
|
|
# @DESCRIPTION: |
13 |
|
|
# The multiprocessing eclass contains a suite of functions that allow ebuilds |
14 |
|
|
# to quickly run things in parallel using shell code. |
15 |
|
|
# |
16 |
|
|
# It has two modes: pre-fork and post-fork. If you don't want to dive into any |
17 |
|
|
# more nuts & bolts, just use the pre-fork mode. For main threads that mostly |
18 |
|
|
# spawn children and then wait for them to finish, use the pre-fork mode. For |
19 |
|
|
# main threads that do a bit of processing themselves, use the post-fork mode. |
20 |
|
|
# You may mix & match them for longer computation loops. |
21 |
|
|
# @EXAMPLE: |
22 |
|
|
# |
23 |
|
|
# @CODE |
24 |
|
|
# # First initialize things: |
25 |
|
|
# multijob_init |
26 |
|
|
# |
27 |
|
|
# # Then hash a bunch of files in parallel: |
28 |
|
|
# for n in {0..20} ; do |
29 |
|
|
# multijob_child_init md5sum data.${n} > data.${n} |
30 |
|
|
# done |
31 |
|
|
# |
32 |
|
|
# # Then wait for all the children to finish: |
33 |
|
|
# multijob_finish |
34 |
|
|
# @CODE |
35 |
|
|
|
36 |
ulm |
1.9 |
if [[ -z ${_MULTIPROCESSING_ECLASS} ]]; then |
37 |
|
|
_MULTIPROCESSING_ECLASS=1 |
38 |
vapier |
1.1 |
|
39 |
vapier |
1.8 |
# @FUNCTION: bashpid |
40 |
|
|
# @DESCRIPTION: |
41 |
|
|
# Return the process id of the current sub shell. This is to support bash |
42 |
|
|
# versions older than 4.0 that lack $BASHPID support natively. Simply do: |
43 |
|
|
# echo ${BASHPID:-$(bashpid)} |
44 |
|
|
# |
45 |
|
|
# Note: Using this func in any other way than the one above is not supported. |
46 |
|
|
bashpid() { |
47 |
|
|
# Running bashpid plainly will return incorrect results. This func must |
48 |
|
|
# be run in a subshell of the current subshell to get the right pid. |
49 |
|
|
# i.e. This will show the wrong value: |
50 |
|
|
# bashpid |
51 |
|
|
# But this will show the right value: |
52 |
|
|
# (bashpid) |
53 |
|
|
sh -c 'echo ${PPID}' |
54 |
|
|
} |
55 |
|
|
|
56 |
vapier |
1.1 |
# @FUNCTION: makeopts_jobs |
57 |
|
|
# @USAGE: [${MAKEOPTS}] |
58 |
|
|
# @DESCRIPTION: |
59 |
|
|
# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number |
60 |
|
|
# specified therein. Useful for running non-make tools in parallel too. |
61 |
|
|
# i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the |
62 |
|
|
# number as bash normalizes it to [0, 255]. If the flags haven't specified a |
63 |
|
|
# -j flag, then "1" is shown as that is the default `make` uses. Since there's |
64 |
|
|
# no way to represent infinity, we return 999 if the user has -j without a number. |
65 |
|
|
makeopts_jobs() { |
66 |
|
|
[[ $# -eq 0 ]] && set -- ${MAKEOPTS} |
67 |
|
|
# This assumes the first .* will be more greedy than the second .* |
68 |
|
|
# since POSIX doesn't specify a non-greedy match (i.e. ".*?"). |
69 |
|
|
local jobs=$(echo " $* " | sed -r -n \ |
70 |
|
|
-e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \ |
71 |
|
|
-e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p') |
72 |
|
|
echo ${jobs:-1} |
73 |
|
|
} |
74 |
|
|
|
75 |
vapier |
1.4 |
# @FUNCTION: makeopts_loadavg |
76 |
|
|
# @USAGE: [${MAKEOPTS}] |
77 |
|
|
# @DESCRIPTION: |
78 |
|
|
# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the value set |
79 |
|
|
# for load-average. For make and ninja based builds this will mean new jobs are |
80 |
|
|
# not only limited by the jobs-value, but also by the current load - which might |
81 |
|
|
# get excessive due to I/O and not just due to CPU load. |
82 |
|
|
# Be aware that the returned number might be a floating-point number. Test |
83 |
|
|
# whether your software supports that. |
84 |
|
|
makeopts_loadavg() { |
85 |
|
|
[[ $# -eq 0 ]] && set -- ${MAKEOPTS} |
86 |
|
|
# This assumes the first .* will be more greedy than the second .* |
87 |
|
|
# since POSIX doesn't specify a non-greedy match (i.e. ".*?"). |
88 |
|
|
local lavg=$(echo " $* " | sed -r -n \ |
89 |
|
|
-e 's:.*[[:space:]](-l|--load-average[=[:space:]])[[:space:]]*([0-9]+|[0-9]+\.[0-9]+)[^0-9.]*:\2:p' \ |
90 |
|
|
-e 's:.*[[:space:]](-l|--load-average)[[:space:]].*:999:p') |
91 |
|
|
echo ${lavg:-1} |
92 |
|
|
} |
93 |
|
|
|
94 |
vapier |
1.1 |
# @FUNCTION: multijob_init |
95 |
|
|
# @USAGE: [${MAKEOPTS}] |
96 |
|
|
# @DESCRIPTION: |
97 |
|
|
# Setup the environment for executing code in parallel. |
98 |
|
|
# You must call this before any other multijob function. |
99 |
|
|
multijob_init() { |
100 |
|
|
# When something goes wrong, try to wait for all the children so we |
101 |
|
|
# don't leave any zombies around. |
102 |
vapier |
1.2 |
has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait " |
103 |
vapier |
1.1 |
|
104 |
|
|
# Setup a pipe for children to write their pids to when they finish. |
105 |
vapier |
1.3 |
# We have to allocate two fd's because POSIX has undefined behavior |
106 |
|
|
# when you open a FIFO for simultaneous read/write. #487056 |
107 |
vapier |
1.1 |
local pipe="${T}/multijob.pipe" |
108 |
vapier |
1.3 |
mkfifo -m 600 "${pipe}" |
109 |
|
|
redirect_alloc_fd mj_write_fd "${pipe}" |
110 |
|
|
redirect_alloc_fd mj_read_fd "${pipe}" |
111 |
vapier |
1.1 |
rm -f "${pipe}" |
112 |
|
|
|
113 |
|
|
# See how many children we can fork based on the user's settings. |
114 |
|
|
mj_max_jobs=$(makeopts_jobs "$@") |
115 |
|
|
mj_num_jobs=0 |
116 |
|
|
} |
117 |
|
|
|
118 |
|
|
# @FUNCTION: multijob_child_init |
119 |
|
|
# @USAGE: [--pre|--post] [command to run in background] |
120 |
|
|
# @DESCRIPTION: |
121 |
|
|
# This function has two forms. You can use it to execute a simple command |
122 |
|
|
# in the background (and it takes care of everything else), or you must |
123 |
|
|
# call this first thing in your forked child process. |
124 |
|
|
# |
125 |
|
|
# The --pre/--post options allow you to select the child generation mode. |
126 |
|
|
# |
127 |
|
|
# @CODE |
128 |
|
|
# # 1st form: pass the command line as arguments: |
129 |
|
|
# multijob_child_init ls /dev |
130 |
|
|
# # Or if you want to use pre/post fork modes: |
131 |
|
|
# multijob_child_init --pre ls /dev |
132 |
|
|
# multijob_child_init --post ls /dev |
133 |
|
|
# |
134 |
|
|
# # 2nd form: execute multiple stuff in the background (post fork): |
135 |
|
|
# ( |
136 |
|
|
# multijob_child_init |
137 |
|
|
# out=`ls` |
138 |
|
|
# if echo "${out}" | grep foo ; then |
139 |
|
|
# echo "YEAH" |
140 |
|
|
# fi |
141 |
|
|
# ) & |
142 |
|
|
# multijob_post_fork |
143 |
|
|
# |
144 |
|
|
# # 2nd form: execute multiple stuff in the background (pre fork): |
145 |
|
|
# multijob_pre_fork |
146 |
|
|
# ( |
147 |
|
|
# multijob_child_init |
148 |
|
|
# out=`ls` |
149 |
|
|
# if echo "${out}" | grep foo ; then |
150 |
|
|
# echo "YEAH" |
151 |
|
|
# fi |
152 |
|
|
# ) & |
153 |
|
|
# @CODE |
154 |
|
|
multijob_child_init() { |
155 |
|
|
local mode="pre" |
156 |
|
|
case $1 in |
157 |
|
|
--pre) mode="pre" ; shift ;; |
158 |
|
|
--post) mode="post"; shift ;; |
159 |
|
|
esac |
160 |
|
|
|
161 |
|
|
if [[ $# -eq 0 ]] ; then |
162 |
vapier |
1.8 |
trap 'echo ${BASHPID:-$(bashpid)} $? >&'${mj_write_fd} EXIT |
163 |
vapier |
1.1 |
trap 'exit 1' INT TERM |
164 |
|
|
else |
165 |
|
|
local ret |
166 |
|
|
[[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; } |
167 |
|
|
( multijob_child_init ; "$@" ) & |
168 |
|
|
[[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; } |
169 |
|
|
return ${ret} |
170 |
|
|
fi |
171 |
|
|
} |
172 |
|
|
|
173 |
|
|
# @FUNCTION: _multijob_fork |
174 |
|
|
# @INTERNAL |
175 |
|
|
# @DESCRIPTION: |
176 |
|
|
# Do the actual book keeping. |
177 |
|
|
_multijob_fork() { |
178 |
|
|
[[ $# -eq 1 ]] || die "incorrect number of arguments" |
179 |
|
|
|
180 |
|
|
local ret=0 |
181 |
|
|
[[ $1 == "post" ]] && : $(( ++mj_num_jobs )) |
182 |
|
|
if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then |
183 |
|
|
multijob_finish_one |
184 |
|
|
ret=$? |
185 |
|
|
fi |
186 |
|
|
[[ $1 == "pre" ]] && : $(( ++mj_num_jobs )) |
187 |
|
|
return ${ret} |
188 |
|
|
} |
189 |
|
|
|
190 |
|
|
# @FUNCTION: multijob_pre_fork |
191 |
|
|
# @DESCRIPTION: |
192 |
|
|
# You must call this in the parent process before forking a child process. |
193 |
|
|
# If the parallel limit has been hit, it will wait for one child to finish |
194 |
|
|
# and return its exit status. |
195 |
|
|
multijob_pre_fork() { _multijob_fork pre "$@" ; } |
196 |
|
|
|
197 |
|
|
# @FUNCTION: multijob_post_fork |
198 |
|
|
# @DESCRIPTION: |
199 |
|
|
# You must call this in the parent process after forking a child process. |
200 |
|
|
# If the parallel limit has been hit, it will wait for one child to finish |
201 |
|
|
# and return its exit status. |
202 |
|
|
multijob_post_fork() { _multijob_fork post "$@" ; } |
203 |
|
|
|
204 |
|
|
# @FUNCTION: multijob_finish_one |
205 |
|
|
# @DESCRIPTION: |
206 |
|
|
# Wait for a single process to exit and return its exit code. |
207 |
|
|
multijob_finish_one() { |
208 |
|
|
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
209 |
|
|
|
210 |
|
|
local pid ret |
211 |
vapier |
1.3 |
read -r -u ${mj_read_fd} pid ret || die |
212 |
vapier |
1.1 |
: $(( --mj_num_jobs )) |
213 |
|
|
return ${ret} |
214 |
|
|
} |
215 |
|
|
|
216 |
|
|
# @FUNCTION: multijob_finish |
217 |
|
|
# @DESCRIPTION: |
218 |
|
|
# Wait for all pending processes to exit and return the bitwise or |
219 |
|
|
# of all their exit codes. |
220 |
|
|
multijob_finish() { |
221 |
|
|
local ret=0 |
222 |
|
|
while [[ ${mj_num_jobs} -gt 0 ]] ; do |
223 |
|
|
multijob_finish_one |
224 |
|
|
: $(( ret |= $? )) |
225 |
|
|
done |
226 |
|
|
# Let bash clean up its internal child tracking state. |
227 |
|
|
wait |
228 |
|
|
|
229 |
|
|
# Do this after reaping all the children. |
230 |
|
|
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
231 |
|
|
|
232 |
vapier |
1.2 |
# No need to hook anymore. |
233 |
|
|
EBUILD_DEATH_HOOKS=${EBUILD_DEATH_HOOKS/ wait / } |
234 |
|
|
|
235 |
vapier |
1.1 |
return ${ret} |
236 |
|
|
} |
237 |
|
|
|
238 |
|
|
# @FUNCTION: redirect_alloc_fd |
239 |
|
|
# @USAGE: <var> <file> [redirection] |
240 |
|
|
# @DESCRIPTION: |
241 |
|
|
# Find a free fd and redirect the specified file via it. Store the new |
242 |
|
|
# fd in the specified variable. Useful for the cases where we don't care |
243 |
|
|
# about the exact fd #. |
244 |
|
|
redirect_alloc_fd() { |
245 |
|
|
local var=$1 file=$2 redir=${3:-"<>"} |
246 |
|
|
|
247 |
jcallen |
1.6 |
# Make sure /dev/fd is sane on Linux hosts. #479656 |
248 |
vapier |
1.7 |
if [[ ! -L /dev/fd && ${CBUILD} == *linux* ]] ; then |
249 |
vapier |
1.5 |
eerror "You're missing a /dev/fd symlink to /proc/self/fd." |
250 |
|
|
eerror "Please fix the symlink and check your boot scripts (udev/etc...)." |
251 |
|
|
die "/dev/fd is broken" |
252 |
|
|
fi |
253 |
|
|
|
254 |
vapier |
1.1 |
if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then |
255 |
|
|
# Newer bash provides this functionality. |
256 |
|
|
eval "exec {${var}}${redir}'${file}'" |
257 |
|
|
else |
258 |
|
|
# Need to provide the functionality ourselves. |
259 |
|
|
local fd=10 |
260 |
|
|
while :; do |
261 |
|
|
# Make sure the fd isn't open. It could be a char device, |
262 |
|
|
# or a symlink (possibly broken) to something else. |
263 |
|
|
if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then |
264 |
|
|
eval "exec ${fd}${redir}'${file}'" && break |
265 |
|
|
fi |
266 |
|
|
[[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?' |
267 |
|
|
: $(( ++fd )) |
268 |
|
|
done |
269 |
|
|
: $(( ${var} = fd )) |
270 |
|
|
fi |
271 |
|
|
} |
272 |
|
|
|
273 |
|
|
fi |