/[gentoo-x86]/eclass/multiprocessing.eclass
Gentoo

Contents of /eclass/multiprocessing.eclass

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.9 - (hide annotations) (download)
Fri Jul 11 08:21:58 2014 UTC (4 months, 2 weeks ago) by ulm
Branch: MAIN
CVS Tags: HEAD
Changes since 1.8: +4 -4 lines
Avoid reserved names for functions and variables, bug 516092.

1 ulm 1.9 # Copyright 1999-2014 Gentoo Foundation
2 vapier 1.1 # Distributed under the terms of the GNU General Public License v2
3 ulm 1.9 # $Header: /var/cvsroot/gentoo-x86/eclass/multiprocessing.eclass,v 1.8 2013/12/21 09:40:37 vapier Exp $
4 vapier 1.1
5     # @ECLASS: multiprocessing.eclass
6     # @MAINTAINER:
7     # base-system@gentoo.org
8     # @AUTHOR:
9     # Brian Harring <ferringb@gentoo.org>
10     # Mike Frysinger <vapier@gentoo.org>
11     # @BLURB: parallelization with bash (wtf?)
12     # @DESCRIPTION:
13     # The multiprocessing eclass contains a suite of functions that allow ebuilds
14     # to quickly run things in parallel using shell code.
15     #
16     # It has two modes: pre-fork and post-fork. If you don't want to dive into any
17     # more nuts & bolts, just use the pre-fork mode. For main threads that mostly
18     # spawn children and then wait for them to finish, use the pre-fork mode. For
19     # main threads that do a bit of processing themselves, use the post-fork mode.
20     # You may mix & match them for longer computation loops.
21     # @EXAMPLE:
22     #
23     # @CODE
24     # # First initialize things:
25     # multijob_init
26     #
27     # # Then hash a bunch of files in parallel:
28     # for n in {0..20} ; do
29     # multijob_child_init md5sum data.${n} > data.${n}
30     # done
31     #
32     # # Then wait for all the children to finish:
33     # multijob_finish
34     # @CODE
35    
36 ulm 1.9 if [[ -z ${_MULTIPROCESSING_ECLASS} ]]; then
37     _MULTIPROCESSING_ECLASS=1
38 vapier 1.1
39 vapier 1.8 # @FUNCTION: bashpid
40     # @DESCRIPTION:
41     # Return the process id of the current sub shell. This is to support bash
42     # versions older than 4.0 that lack $BASHPID support natively. Simply do:
43     # echo ${BASHPID:-$(bashpid)}
44     #
45     # Note: Using this func in any other way than the one above is not supported.
46     bashpid() {
47     # Running bashpid plainly will return incorrect results. This func must
48     # be run in a subshell of the current subshell to get the right pid.
49     # i.e. This will show the wrong value:
50     # bashpid
51     # But this will show the right value:
52     # (bashpid)
53     sh -c 'echo ${PPID}'
54     }
55    
56 vapier 1.1 # @FUNCTION: makeopts_jobs
57     # @USAGE: [${MAKEOPTS}]
58     # @DESCRIPTION:
59     # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
60     # specified therein. Useful for running non-make tools in parallel too.
61     # i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
62     # number as bash normalizes it to [0, 255]. If the flags haven't specified a
63     # -j flag, then "1" is shown as that is the default `make` uses. Since there's
64     # no way to represent infinity, we return 999 if the user has -j without a number.
65     makeopts_jobs() {
66     [[ $# -eq 0 ]] && set -- ${MAKEOPTS}
67     # This assumes the first .* will be more greedy than the second .*
68     # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
69     local jobs=$(echo " $* " | sed -r -n \
70     -e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \
71     -e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p')
72     echo ${jobs:-1}
73     }
74    
75 vapier 1.4 # @FUNCTION: makeopts_loadavg
76     # @USAGE: [${MAKEOPTS}]
77     # @DESCRIPTION:
78     # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the value set
79     # for load-average. For make and ninja based builds this will mean new jobs are
80     # not only limited by the jobs-value, but also by the current load - which might
81     # get excessive due to I/O and not just due to CPU load.
82     # Be aware that the returned number might be a floating-point number. Test
83     # whether your software supports that.
84     makeopts_loadavg() {
85     [[ $# -eq 0 ]] && set -- ${MAKEOPTS}
86     # This assumes the first .* will be more greedy than the second .*
87     # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
88     local lavg=$(echo " $* " | sed -r -n \
89     -e 's:.*[[:space:]](-l|--load-average[=[:space:]])[[:space:]]*([0-9]+|[0-9]+\.[0-9]+)[^0-9.]*:\2:p' \
90     -e 's:.*[[:space:]](-l|--load-average)[[:space:]].*:999:p')
91     echo ${lavg:-1}
92     }
93    
94 vapier 1.1 # @FUNCTION: multijob_init
95     # @USAGE: [${MAKEOPTS}]
96     # @DESCRIPTION:
97     # Setup the environment for executing code in parallel.
98     # You must call this before any other multijob function.
99     multijob_init() {
100     # When something goes wrong, try to wait for all the children so we
101     # don't leave any zombies around.
102 vapier 1.2 has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait "
103 vapier 1.1
104     # Setup a pipe for children to write their pids to when they finish.
105 vapier 1.3 # We have to allocate two fd's because POSIX has undefined behavior
106     # when you open a FIFO for simultaneous read/write. #487056
107 vapier 1.1 local pipe="${T}/multijob.pipe"
108 vapier 1.3 mkfifo -m 600 "${pipe}"
109     redirect_alloc_fd mj_write_fd "${pipe}"
110     redirect_alloc_fd mj_read_fd "${pipe}"
111 vapier 1.1 rm -f "${pipe}"
112    
113     # See how many children we can fork based on the user's settings.
114     mj_max_jobs=$(makeopts_jobs "$@")
115     mj_num_jobs=0
116     }
117    
118     # @FUNCTION: multijob_child_init
119     # @USAGE: [--pre|--post] [command to run in background]
120     # @DESCRIPTION:
121     # This function has two forms. You can use it to execute a simple command
122     # in the background (and it takes care of everything else), or you must
123     # call this first thing in your forked child process.
124     #
125     # The --pre/--post options allow you to select the child generation mode.
126     #
127     # @CODE
128     # # 1st form: pass the command line as arguments:
129     # multijob_child_init ls /dev
130     # # Or if you want to use pre/post fork modes:
131     # multijob_child_init --pre ls /dev
132     # multijob_child_init --post ls /dev
133     #
134     # # 2nd form: execute multiple stuff in the background (post fork):
135     # (
136     # multijob_child_init
137     # out=`ls`
138     # if echo "${out}" | grep foo ; then
139     # echo "YEAH"
140     # fi
141     # ) &
142     # multijob_post_fork
143     #
144     # # 2nd form: execute multiple stuff in the background (pre fork):
145     # multijob_pre_fork
146     # (
147     # multijob_child_init
148     # out=`ls`
149     # if echo "${out}" | grep foo ; then
150     # echo "YEAH"
151     # fi
152     # ) &
153     # @CODE
154     multijob_child_init() {
155     local mode="pre"
156     case $1 in
157     --pre) mode="pre" ; shift ;;
158     --post) mode="post"; shift ;;
159     esac
160    
161     if [[ $# -eq 0 ]] ; then
162 vapier 1.8 trap 'echo ${BASHPID:-$(bashpid)} $? >&'${mj_write_fd} EXIT
163 vapier 1.1 trap 'exit 1' INT TERM
164     else
165     local ret
166     [[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; }
167     ( multijob_child_init ; "$@" ) &
168     [[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; }
169     return ${ret}
170     fi
171     }
172    
173     # @FUNCTION: _multijob_fork
174     # @INTERNAL
175     # @DESCRIPTION:
176     # Do the actual book keeping.
177     _multijob_fork() {
178     [[ $# -eq 1 ]] || die "incorrect number of arguments"
179    
180     local ret=0
181     [[ $1 == "post" ]] && : $(( ++mj_num_jobs ))
182     if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then
183     multijob_finish_one
184     ret=$?
185     fi
186     [[ $1 == "pre" ]] && : $(( ++mj_num_jobs ))
187     return ${ret}
188     }
189    
190     # @FUNCTION: multijob_pre_fork
191     # @DESCRIPTION:
192     # You must call this in the parent process before forking a child process.
193     # If the parallel limit has been hit, it will wait for one child to finish
194     # and return its exit status.
195     multijob_pre_fork() { _multijob_fork pre "$@" ; }
196    
197     # @FUNCTION: multijob_post_fork
198     # @DESCRIPTION:
199     # You must call this in the parent process after forking a child process.
200     # If the parallel limit has been hit, it will wait for one child to finish
201     # and return its exit status.
202     multijob_post_fork() { _multijob_fork post "$@" ; }
203    
204     # @FUNCTION: multijob_finish_one
205     # @DESCRIPTION:
206     # Wait for a single process to exit and return its exit code.
207     multijob_finish_one() {
208     [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
209    
210     local pid ret
211 vapier 1.3 read -r -u ${mj_read_fd} pid ret || die
212 vapier 1.1 : $(( --mj_num_jobs ))
213     return ${ret}
214     }
215    
216     # @FUNCTION: multijob_finish
217     # @DESCRIPTION:
218     # Wait for all pending processes to exit and return the bitwise or
219     # of all their exit codes.
220     multijob_finish() {
221     local ret=0
222     while [[ ${mj_num_jobs} -gt 0 ]] ; do
223     multijob_finish_one
224     : $(( ret |= $? ))
225     done
226     # Let bash clean up its internal child tracking state.
227     wait
228    
229     # Do this after reaping all the children.
230     [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
231    
232 vapier 1.2 # No need to hook anymore.
233     EBUILD_DEATH_HOOKS=${EBUILD_DEATH_HOOKS/ wait / }
234    
235 vapier 1.1 return ${ret}
236     }
237    
238     # @FUNCTION: redirect_alloc_fd
239     # @USAGE: <var> <file> [redirection]
240     # @DESCRIPTION:
241     # Find a free fd and redirect the specified file via it. Store the new
242     # fd in the specified variable. Useful for the cases where we don't care
243     # about the exact fd #.
244     redirect_alloc_fd() {
245     local var=$1 file=$2 redir=${3:-"<>"}
246    
247 jcallen 1.6 # Make sure /dev/fd is sane on Linux hosts. #479656
248 vapier 1.7 if [[ ! -L /dev/fd && ${CBUILD} == *linux* ]] ; then
249 vapier 1.5 eerror "You're missing a /dev/fd symlink to /proc/self/fd."
250     eerror "Please fix the symlink and check your boot scripts (udev/etc...)."
251     die "/dev/fd is broken"
252     fi
253    
254 vapier 1.1 if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then
255     # Newer bash provides this functionality.
256     eval "exec {${var}}${redir}'${file}'"
257     else
258     # Need to provide the functionality ourselves.
259     local fd=10
260     while :; do
261     # Make sure the fd isn't open. It could be a char device,
262     # or a symlink (possibly broken) to something else.
263     if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then
264     eval "exec ${fd}${redir}'${file}'" && break
265     fi
266     [[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?'
267     : $(( ++fd ))
268     done
269     : $(( ${var} = fd ))
270     fi
271     }
272    
273     fi

  ViewVC Help
Powered by ViewVC 1.1.20