/[gentoo-x86]/eclass/multiprocessing.eclass
Gentoo

Contents of /eclass/multiprocessing.eclass

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.1 - (show annotations) (download)
Thu Jun 7 04:59:40 2012 UTC (22 months, 2 weeks ago) by vapier
Branch: MAIN
initial multiprocessing eclass

1 # Copyright 1999-2012 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
3 # $Header: $
4
5 # @ECLASS: multiprocessing.eclass
6 # @MAINTAINER:
7 # base-system@gentoo.org
8 # @AUTHOR:
9 # Brian Harring <ferringb@gentoo.org>
10 # Mike Frysinger <vapier@gentoo.org>
11 # @BLURB: parallelization with bash (wtf?)
12 # @DESCRIPTION:
13 # The multiprocessing eclass contains a suite of functions that allow ebuilds
14 # to quickly run things in parallel using shell code.
15 #
16 # It has two modes: pre-fork and post-fork. If you don't want to dive into any
17 # more nuts & bolts, just use the pre-fork mode. For main threads that mostly
18 # spawn children and then wait for them to finish, use the pre-fork mode. For
19 # main threads that do a bit of processing themselves, use the post-fork mode.
20 # You may mix & match them for longer computation loops.
21 # @EXAMPLE:
22 #
23 # @CODE
24 # # First initialize things:
25 # multijob_init
26 #
27 # # Then hash a bunch of files in parallel:
28 # for n in {0..20} ; do
29 # multijob_child_init md5sum data.${n} > data.${n}
30 # done
31 #
32 # # Then wait for all the children to finish:
33 # multijob_finish
34 # @CODE
35
36 if [[ ${___ECLASS_ONCE_MULTIPROCESSING} != "recur -_+^+_- spank" ]] ; then
37 ___ECLASS_ONCE_MULTIPROCESSING="recur -_+^+_- spank"
38
39 # @FUNCTION: makeopts_jobs
40 # @USAGE: [${MAKEOPTS}]
41 # @DESCRIPTION:
42 # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
43 # specified therein. Useful for running non-make tools in parallel too.
44 # i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
45 # number as bash normalizes it to [0, 255]. If the flags haven't specified a
46 # -j flag, then "1" is shown as that is the default `make` uses. Since there's
47 # no way to represent infinity, we return 999 if the user has -j without a number.
48 makeopts_jobs() {
49 [[ $# -eq 0 ]] && set -- ${MAKEOPTS}
50 # This assumes the first .* will be more greedy than the second .*
51 # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
52 local jobs=$(echo " $* " | sed -r -n \
53 -e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \
54 -e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p')
55 echo ${jobs:-1}
56 }
57
58 # @FUNCTION: multijob_init
59 # @USAGE: [${MAKEOPTS}]
60 # @DESCRIPTION:
61 # Setup the environment for executing code in parallel.
62 # You must call this before any other multijob function.
63 multijob_init() {
64 # When something goes wrong, try to wait for all the children so we
65 # don't leave any zombies around.
66 has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait"
67
68 # Setup a pipe for children to write their pids to when they finish.
69 local pipe="${T}/multijob.pipe"
70 mkfifo "${pipe}"
71 redirect_alloc_fd mj_control_fd "${pipe}"
72 rm -f "${pipe}"
73
74 # See how many children we can fork based on the user's settings.
75 mj_max_jobs=$(makeopts_jobs "$@")
76 mj_num_jobs=0
77 }
78
79 # @FUNCTION: multijob_child_init
80 # @USAGE: [--pre|--post] [command to run in background]
81 # @DESCRIPTION:
82 # This function has two forms. You can use it to execute a simple command
83 # in the background (and it takes care of everything else), or you must
84 # call this first thing in your forked child process.
85 #
86 # The --pre/--post options allow you to select the child generation mode.
87 #
88 # @CODE
89 # # 1st form: pass the command line as arguments:
90 # multijob_child_init ls /dev
91 # # Or if you want to use pre/post fork modes:
92 # multijob_child_init --pre ls /dev
93 # multijob_child_init --post ls /dev
94 #
95 # # 2nd form: execute multiple stuff in the background (post fork):
96 # (
97 # multijob_child_init
98 # out=`ls`
99 # if echo "${out}" | grep foo ; then
100 # echo "YEAH"
101 # fi
102 # ) &
103 # multijob_post_fork
104 #
105 # # 2nd form: execute multiple stuff in the background (pre fork):
106 # multijob_pre_fork
107 # (
108 # multijob_child_init
109 # out=`ls`
110 # if echo "${out}" | grep foo ; then
111 # echo "YEAH"
112 # fi
113 # ) &
114 # @CODE
115 multijob_child_init() {
116 local mode="pre"
117 case $1 in
118 --pre) mode="pre" ; shift ;;
119 --post) mode="post"; shift ;;
120 esac
121
122 if [[ $# -eq 0 ]] ; then
123 trap 'echo ${BASHPID} $? >&'${mj_control_fd} EXIT
124 trap 'exit 1' INT TERM
125 else
126 local ret
127 [[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; }
128 ( multijob_child_init ; "$@" ) &
129 [[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; }
130 return ${ret}
131 fi
132 }
133
134 # @FUNCTION: _multijob_fork
135 # @INTERNAL
136 # @DESCRIPTION:
137 # Do the actual book keeping.
138 _multijob_fork() {
139 [[ $# -eq 1 ]] || die "incorrect number of arguments"
140
141 local ret=0
142 [[ $1 == "post" ]] && : $(( ++mj_num_jobs ))
143 if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then
144 multijob_finish_one
145 ret=$?
146 fi
147 [[ $1 == "pre" ]] && : $(( ++mj_num_jobs ))
148 return ${ret}
149 }
150
151 # @FUNCTION: multijob_pre_fork
152 # @DESCRIPTION:
153 # You must call this in the parent process before forking a child process.
154 # If the parallel limit has been hit, it will wait for one child to finish
155 # and return its exit status.
156 multijob_pre_fork() { _multijob_fork pre "$@" ; }
157
158 # @FUNCTION: multijob_post_fork
159 # @DESCRIPTION:
160 # You must call this in the parent process after forking a child process.
161 # If the parallel limit has been hit, it will wait for one child to finish
162 # and return its exit status.
163 multijob_post_fork() { _multijob_fork post "$@" ; }
164
165 # @FUNCTION: multijob_finish_one
166 # @DESCRIPTION:
167 # Wait for a single process to exit and return its exit code.
168 multijob_finish_one() {
169 [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
170
171 local pid ret
172 read -r -u ${mj_control_fd} pid ret || die
173 : $(( --mj_num_jobs ))
174 return ${ret}
175 }
176
177 # @FUNCTION: multijob_finish
178 # @DESCRIPTION:
179 # Wait for all pending processes to exit and return the bitwise or
180 # of all their exit codes.
181 multijob_finish() {
182 local ret=0
183 while [[ ${mj_num_jobs} -gt 0 ]] ; do
184 multijob_finish_one
185 : $(( ret |= $? ))
186 done
187 # Let bash clean up its internal child tracking state.
188 wait
189
190 # Do this after reaping all the children.
191 [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
192
193 return ${ret}
194 }
195
196 # @FUNCTION: redirect_alloc_fd
197 # @USAGE: <var> <file> [redirection]
198 # @DESCRIPTION:
199 # Find a free fd and redirect the specified file via it. Store the new
200 # fd in the specified variable. Useful for the cases where we don't care
201 # about the exact fd #.
202 redirect_alloc_fd() {
203 local var=$1 file=$2 redir=${3:-"<>"}
204
205 if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then
206 # Newer bash provides this functionality.
207 eval "exec {${var}}${redir}'${file}'"
208 else
209 # Need to provide the functionality ourselves.
210 local fd=10
211 while :; do
212 # Make sure the fd isn't open. It could be a char device,
213 # or a symlink (possibly broken) to something else.
214 if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then
215 eval "exec ${fd}${redir}'${file}'" && break
216 fi
217 [[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?'
218 : $(( ++fd ))
219 done
220 : $(( ${var} = fd ))
221 fi
222 }
223
224 fi

  ViewVC Help
Powered by ViewVC 1.1.20