/[gentoo-x86]/eclass/multiprocessing.eclass
Gentoo

Contents of /eclass/multiprocessing.eclass

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.3 - (hide annotations) (download)
Sat Oct 12 21:12:48 2013 UTC (13 months, 2 weeks ago) by vapier
Branch: MAIN
Changes since 1.2: +9 -6 lines
split the FIFO fd into two (one ro and one rw) to avoid POSIX undefined behavior #487056 by Alan Hourihane

1 vapier 1.3 # Copyright 1999-2013 Gentoo Foundation
2 vapier 1.1 # Distributed under the terms of the GNU General Public License v2
3 vapier 1.3 # $Header: /var/cvsroot/gentoo-x86/eclass/multiprocessing.eclass,v 1.2 2012/07/30 14:52:18 vapier Exp $
4 vapier 1.1
5     # @ECLASS: multiprocessing.eclass
6     # @MAINTAINER:
7     # base-system@gentoo.org
8     # @AUTHOR:
9     # Brian Harring <ferringb@gentoo.org>
10     # Mike Frysinger <vapier@gentoo.org>
11     # @BLURB: parallelization with bash (wtf?)
12     # @DESCRIPTION:
13     # The multiprocessing eclass contains a suite of functions that allow ebuilds
14     # to quickly run things in parallel using shell code.
15     #
16     # It has two modes: pre-fork and post-fork. If you don't want to dive into any
17     # more nuts & bolts, just use the pre-fork mode. For main threads that mostly
18     # spawn children and then wait for them to finish, use the pre-fork mode. For
19     # main threads that do a bit of processing themselves, use the post-fork mode.
20     # You may mix & match them for longer computation loops.
21     # @EXAMPLE:
22     #
23     # @CODE
24     # # First initialize things:
25     # multijob_init
26     #
27     # # Then hash a bunch of files in parallel:
28     # for n in {0..20} ; do
29     # multijob_child_init md5sum data.${n} > data.${n}
30     # done
31     #
32     # # Then wait for all the children to finish:
33     # multijob_finish
34     # @CODE
35    
36     if [[ ${___ECLASS_ONCE_MULTIPROCESSING} != "recur -_+^+_- spank" ]] ; then
37     ___ECLASS_ONCE_MULTIPROCESSING="recur -_+^+_- spank"
38    
39     # @FUNCTION: makeopts_jobs
40     # @USAGE: [${MAKEOPTS}]
41     # @DESCRIPTION:
42     # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
43     # specified therein. Useful for running non-make tools in parallel too.
44     # i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
45     # number as bash normalizes it to [0, 255]. If the flags haven't specified a
46     # -j flag, then "1" is shown as that is the default `make` uses. Since there's
47     # no way to represent infinity, we return 999 if the user has -j without a number.
48     makeopts_jobs() {
49     [[ $# -eq 0 ]] && set -- ${MAKEOPTS}
50     # This assumes the first .* will be more greedy than the second .*
51     # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
52     local jobs=$(echo " $* " | sed -r -n \
53     -e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \
54     -e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p')
55     echo ${jobs:-1}
56     }
57    
58     # @FUNCTION: multijob_init
59     # @USAGE: [${MAKEOPTS}]
60     # @DESCRIPTION:
61     # Setup the environment for executing code in parallel.
62     # You must call this before any other multijob function.
63     multijob_init() {
64     # When something goes wrong, try to wait for all the children so we
65     # don't leave any zombies around.
66 vapier 1.2 has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait "
67 vapier 1.1
68     # Setup a pipe for children to write their pids to when they finish.
69 vapier 1.3 # We have to allocate two fd's because POSIX has undefined behavior
70     # when you open a FIFO for simultaneous read/write. #487056
71 vapier 1.1 local pipe="${T}/multijob.pipe"
72 vapier 1.3 mkfifo -m 600 "${pipe}"
73     redirect_alloc_fd mj_write_fd "${pipe}"
74     redirect_alloc_fd mj_read_fd "${pipe}"
75 vapier 1.1 rm -f "${pipe}"
76    
77     # See how many children we can fork based on the user's settings.
78     mj_max_jobs=$(makeopts_jobs "$@")
79     mj_num_jobs=0
80     }
81    
82     # @FUNCTION: multijob_child_init
83     # @USAGE: [--pre|--post] [command to run in background]
84     # @DESCRIPTION:
85     # This function has two forms. You can use it to execute a simple command
86     # in the background (and it takes care of everything else), or you must
87     # call this first thing in your forked child process.
88     #
89     # The --pre/--post options allow you to select the child generation mode.
90     #
91     # @CODE
92     # # 1st form: pass the command line as arguments:
93     # multijob_child_init ls /dev
94     # # Or if you want to use pre/post fork modes:
95     # multijob_child_init --pre ls /dev
96     # multijob_child_init --post ls /dev
97     #
98     # # 2nd form: execute multiple stuff in the background (post fork):
99     # (
100     # multijob_child_init
101     # out=`ls`
102     # if echo "${out}" | grep foo ; then
103     # echo "YEAH"
104     # fi
105     # ) &
106     # multijob_post_fork
107     #
108     # # 2nd form: execute multiple stuff in the background (pre fork):
109     # multijob_pre_fork
110     # (
111     # multijob_child_init
112     # out=`ls`
113     # if echo "${out}" | grep foo ; then
114     # echo "YEAH"
115     # fi
116     # ) &
117     # @CODE
118     multijob_child_init() {
119     local mode="pre"
120     case $1 in
121     --pre) mode="pre" ; shift ;;
122     --post) mode="post"; shift ;;
123     esac
124    
125     if [[ $# -eq 0 ]] ; then
126 vapier 1.3 trap 'echo ${BASHPID} $? >&'${mj_write_fd} EXIT
127 vapier 1.1 trap 'exit 1' INT TERM
128     else
129     local ret
130     [[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; }
131     ( multijob_child_init ; "$@" ) &
132     [[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; }
133     return ${ret}
134     fi
135     }
136    
137     # @FUNCTION: _multijob_fork
138     # @INTERNAL
139     # @DESCRIPTION:
140     # Do the actual book keeping.
141     _multijob_fork() {
142     [[ $# -eq 1 ]] || die "incorrect number of arguments"
143    
144     local ret=0
145     [[ $1 == "post" ]] && : $(( ++mj_num_jobs ))
146     if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then
147     multijob_finish_one
148     ret=$?
149     fi
150     [[ $1 == "pre" ]] && : $(( ++mj_num_jobs ))
151     return ${ret}
152     }
153    
154     # @FUNCTION: multijob_pre_fork
155     # @DESCRIPTION:
156     # You must call this in the parent process before forking a child process.
157     # If the parallel limit has been hit, it will wait for one child to finish
158     # and return its exit status.
159     multijob_pre_fork() { _multijob_fork pre "$@" ; }
160    
161     # @FUNCTION: multijob_post_fork
162     # @DESCRIPTION:
163     # You must call this in the parent process after forking a child process.
164     # If the parallel limit has been hit, it will wait for one child to finish
165     # and return its exit status.
166     multijob_post_fork() { _multijob_fork post "$@" ; }
167    
168     # @FUNCTION: multijob_finish_one
169     # @DESCRIPTION:
170     # Wait for a single process to exit and return its exit code.
171     multijob_finish_one() {
172     [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
173    
174     local pid ret
175 vapier 1.3 read -r -u ${mj_read_fd} pid ret || die
176 vapier 1.1 : $(( --mj_num_jobs ))
177     return ${ret}
178     }
179    
180     # @FUNCTION: multijob_finish
181     # @DESCRIPTION:
182     # Wait for all pending processes to exit and return the bitwise or
183     # of all their exit codes.
184     multijob_finish() {
185     local ret=0
186     while [[ ${mj_num_jobs} -gt 0 ]] ; do
187     multijob_finish_one
188     : $(( ret |= $? ))
189     done
190     # Let bash clean up its internal child tracking state.
191     wait
192    
193     # Do this after reaping all the children.
194     [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
195    
196 vapier 1.2 # No need to hook anymore.
197     EBUILD_DEATH_HOOKS=${EBUILD_DEATH_HOOKS/ wait / }
198    
199 vapier 1.1 return ${ret}
200     }
201    
202     # @FUNCTION: redirect_alloc_fd
203     # @USAGE: <var> <file> [redirection]
204     # @DESCRIPTION:
205     # Find a free fd and redirect the specified file via it. Store the new
206     # fd in the specified variable. Useful for the cases where we don't care
207     # about the exact fd #.
208     redirect_alloc_fd() {
209     local var=$1 file=$2 redir=${3:-"<>"}
210    
211     if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then
212     # Newer bash provides this functionality.
213     eval "exec {${var}}${redir}'${file}'"
214     else
215     # Need to provide the functionality ourselves.
216     local fd=10
217     while :; do
218     # Make sure the fd isn't open. It could be a char device,
219     # or a symlink (possibly broken) to something else.
220     if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then
221     eval "exec ${fd}${redir}'${file}'" && break
222     fi
223     [[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?'
224     : $(( ++fd ))
225     done
226     : $(( ${var} = fd ))
227     fi
228     }
229    
230     fi

  ViewVC Help
Powered by ViewVC 1.1.20