gentoo-x86/eclass/multiprocessing.eclass

# Copyright 1999-2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: /var/cvsroot/gentoo-x86/eclass/multiprocessing.eclass,v 1.8 2013/12/21 09:40:37 vapier Exp $

# @ECLASS: multiprocessing.eclass
# @MAINTAINER:
# base-system@gentoo.org
# @AUTHOR:
# Brian Harring <ferringb@gentoo.org>
# Mike Frysinger <vapier@gentoo.org>
# @BLURB: parallelization with bash (wtf?)
# @DESCRIPTION:
# The multiprocessing eclass contains a suite of functions that allow ebuilds
# to quickly run things in parallel using shell code.
#
# It has two modes: pre-fork and post-fork.  If you don't want to dive into any
# more nuts & bolts, just use the pre-fork mode.  For main threads that mostly
# spawn children and then wait for them to finish, use the pre-fork mode.  For
# main threads that do a bit of processing themselves, use the post-fork mode.
# You may mix & match them for longer computation loops.
# @EXAMPLE:
#
# @CODE
# # First initialize things:
# multijob_init
#
# # Then hash a bunch of files in parallel:
# for n in {0..20} ; do
#       multijob_child_init md5sum data.${n} > data.${n}
# done
#
# # Then wait for all the children to finish:
# multijob_finish
# @CODE

if [[ -z ${_MULTIPROCESSING_ECLASS} ]]; then
_MULTIPROCESSING_ECLASS=1

# @FUNCTION: bashpid
# @DESCRIPTION:
# Return the process id of the current sub shell.  This is to support bash
# versions older than 4.0 that lack $BASHPID support natively.  Simply do:
# echo ${BASHPID:-$(bashpid)}
#
# Note: Using this func in any other way than the one above is not supported.
bashpid() {
        # Running bashpid plainly will return incorrect results.  This func must
        # be run in a subshell of the current subshell to get the right pid.
        # i.e. This will show the wrong value:
        #   bashpid
        # But this will show the right value:
        #   (bashpid)
        sh -c 'echo ${PPID}'
}

# @FUNCTION: makeopts_jobs
# @USAGE: [${MAKEOPTS}]
# @DESCRIPTION:
# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
# specified therein.  Useful for running non-make tools in parallel too.
# i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
# number as bash normalizes it to [0, 255].  If the flags haven't specified a
# -j flag, then "1" is shown as that is the default `make` uses.  Since there's
# no way to represent infinity, we return 999 if the user has -j without a number.
makeopts_jobs() {
        [[ $# -eq 0 ]] && set -- ${MAKEOPTS}
        # This assumes the first .* will be more greedy than the second .*
        # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
        local jobs=$(echo " $* " | sed -r -n \
                -e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \
                -e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p')
        echo ${jobs:-1}
}

# @FUNCTION: makeopts_loadavg
# @USAGE: [${MAKEOPTS}]
# @DESCRIPTION:
# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the value set
# for load-average. For make and ninja based builds this will mean new jobs are
# not only limited by the jobs-value, but also by the current load - which might
# get excessive due to I/O and not just due to CPU load.
# Be aware that the returned number might be a floating-point number. Test
# whether your software supports that.
makeopts_loadavg() {
        [[ $# -eq 0 ]] && set -- ${MAKEOPTS}
        # This assumes the first .* will be more greedy than the second .*
        # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
        local lavg=$(echo " $* " | sed -r -n \
                -e 's:.*[[:space:]](-l|--load-average[=[:space:]])[[:space:]]*([0-9]+|[0-9]+\.[0-9]+)[^0-9.]*:\2:p' \
                -e 's:.*[[:space:]](-l|--load-average)[[:space:]].*:999:p')
        echo ${lavg:-1}
}

# @FUNCTION: multijob_init
# @USAGE: [${MAKEOPTS}]
# @DESCRIPTION:
# Setup the environment for executing code in parallel.
# You must call this before any other multijob function.
multijob_init() {
        # When something goes wrong, try to wait for all the children so we
        # don't leave any zombies around.
        has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait "

        # Setup a pipe for children to write their pids to when they finish.
        # We have to allocate two fd's because POSIX has undefined behavior
        # when you open a FIFO for simultaneous read/write. #487056
        local pipe="${T}/multijob.pipe"
        mkfifo -m 600 "${pipe}"
        redirect_alloc_fd mj_write_fd "${pipe}"
        redirect_alloc_fd mj_read_fd "${pipe}"
        rm -f "${pipe}"

        # See how many children we can fork based on the user's settings.
        mj_max_jobs=$(makeopts_jobs "$@")
        mj_num_jobs=0
}

# @FUNCTION: multijob_child_init
# @USAGE: [--pre|--post] [command to run in background]
# @DESCRIPTION:
# This function has two forms.  You can use it to execute a simple command
# in the background (and it takes care of everything else), or you must
# call this first thing in your forked child process.
#
# The --pre/--post options allow you to select the child generation mode.
#
# @CODE
# # 1st form: pass the command line as arguments:
# multijob_child_init ls /dev
# # Or if you want to use pre/post fork modes:
# multijob_child_init --pre ls /dev
# multijob_child_init --post ls /dev
#
# # 2nd form: execute multiple stuff in the background (post fork):
# (
# multijob_child_init
# out=`ls`
# if echo "${out}" | grep foo ; then
#       echo "YEAH"
# fi
# ) &
# multijob_post_fork
#
# # 2nd form: execute multiple stuff in the background (pre fork):
# multijob_pre_fork
# (
# multijob_child_init
# out=`ls`
# if echo "${out}" | grep foo ; then
#       echo "YEAH"
# fi
# ) &
# @CODE
multijob_child_init() {
        local mode="pre"
        case $1 in
        --pre)  mode="pre" ; shift ;;
        --post) mode="post"; shift ;;
        esac

        if [[ $# -eq 0 ]] ; then
                trap 'echo ${BASHPID:-$(bashpid)} $? >&'${mj_write_fd} EXIT
                trap 'exit 1' INT TERM
        else
                local ret
                [[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; }
                ( multijob_child_init ; "$@" ) &
                [[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; }
                return ${ret}
        fi
}

# @FUNCTION: _multijob_fork
# @INTERNAL
# @DESCRIPTION:
# Do the actual book keeping.
_multijob_fork() {
        [[ $# -eq 1 ]] || die "incorrect number of arguments"

        local ret=0
        [[ $1 == "post" ]] && : $(( ++mj_num_jobs ))
        if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then
                multijob_finish_one
                ret=$?
        fi
        [[ $1 == "pre" ]] && : $(( ++mj_num_jobs ))
        return ${ret}
}

# @FUNCTION: multijob_pre_fork
# @DESCRIPTION:
# You must call this in the parent process before forking a child process.
# If the parallel limit has been hit, it will wait for one child to finish
# and return its exit status.
multijob_pre_fork() { _multijob_fork pre "$@" ; }

# @FUNCTION: multijob_post_fork
# @DESCRIPTION:
# You must call this in the parent process after forking a child process.
# If the parallel limit has been hit, it will wait for one child to finish
# and return its exit status.
multijob_post_fork() { _multijob_fork post "$@" ; }

# @FUNCTION: multijob_finish_one
# @DESCRIPTION:
# Wait for a single process to exit and return its exit code.
multijob_finish_one() {
        [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"

        local pid ret
        read -r -u ${mj_read_fd} pid ret || die
        : $(( --mj_num_jobs ))
        return ${ret}
}

# @FUNCTION: multijob_finish
# @DESCRIPTION:
# Wait for all pending processes to exit and return the bitwise or
# of all their exit codes.
multijob_finish() {
        local ret=0
        while [[ ${mj_num_jobs} -gt 0 ]] ; do
                multijob_finish_one
                : $(( ret |= $? ))
        done
        # Let bash clean up its internal child tracking state.
        wait

        # Do this after reaping all the children.
        [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"

        # No need to hook anymore.
        EBUILD_DEATH_HOOKS=${EBUILD_DEATH_HOOKS/ wait / }

        return ${ret}
}

# @FUNCTION: redirect_alloc_fd
# @USAGE: <var> <file> [redirection]
# @DESCRIPTION:
# Find a free fd and redirect the specified file via it.  Store the new
# fd in the specified variable.  Useful for the cases where we don't care
# about the exact fd #.
redirect_alloc_fd() {
        local var=$1 file=$2 redir=${3:-"<>"}

        # Make sure /dev/fd is sane on Linux hosts. #479656
        if [[ ! -L /dev/fd && ${CBUILD} == *linux* ]] ; then
                eerror "You're missing a /dev/fd symlink to /proc/self/fd."
                eerror "Please fix the symlink and check your boot scripts (udev/etc...)."
                die "/dev/fd is broken"
        fi

        if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then
                # Newer bash provides this functionality.
                eval "exec {${var}}${redir}'${file}'"
        else
                # Need to provide the functionality ourselves.
                local fd=10
                while :; do
                        # Make sure the fd isn't open.  It could be a char device,
                        # or a symlink (possibly broken) to something else.
                        if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then
                                eval "exec ${fd}${redir}'${file}'" && break
                        fi
                        [[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?'
                        : $(( ++fd ))
                done
                : $(( ${var} = fd ))
        fi
}

fi
1	ulm	1.9	# Copyright 1999-2014 Gentoo Foundation
2	vapier	1.1	# Distributed under the terms of the GNU General Public License v2
3	ulm	1.9	# $Header: /var/cvsroot/gentoo-x86/eclass/multiprocessing.eclass,v 1.8 2013/12/21 09:40:37 vapier Exp $
4	vapier	1.1
5			# @ECLASS: multiprocessing.eclass
6			# @MAINTAINER:
7			# base-system@gentoo.org
8			# @AUTHOR:
9			# Brian Harring <ferringb@gentoo.org>
10			# Mike Frysinger <vapier@gentoo.org>
11			# @BLURB: parallelization with bash (wtf?)
12			# @DESCRIPTION:
13			# The multiprocessing eclass contains a suite of functions that allow ebuilds
14			# to quickly run things in parallel using shell code.
15			#
16			# It has two modes: pre-fork and post-fork. If you don't want to dive into any
17			# more nuts & bolts, just use the pre-fork mode. For main threads that mostly
18			# spawn children and then wait for them to finish, use the pre-fork mode. For
19			# main threads that do a bit of processing themselves, use the post-fork mode.
20			# You may mix & match them for longer computation loops.
21			# @EXAMPLE:
22			#
23			# @CODE
24			# # First initialize things:
25			# multijob_init
26			#
27			# # Then hash a bunch of files in parallel:
28			# for n in {0..20} ; do
29			# multijob_child_init md5sum data.${n} > data.${n}
30			# done
31			#
32			# # Then wait for all the children to finish:
33			# multijob_finish
34			# @CODE
35
36	ulm	1.9	if [[ -z ${_MULTIPROCESSING_ECLASS} ]]; then
37			_MULTIPROCESSING_ECLASS=1
38	vapier	1.1
39	vapier	1.8	# @FUNCTION: bashpid
40			# @DESCRIPTION:
41			# Return the process id of the current sub shell. This is to support bash
42			# versions older than 4.0 that lack $BASHPID support natively. Simply do:
43			# echo ${BASHPID:-$(bashpid)}
44			#
45			# Note: Using this func in any other way than the one above is not supported.
46			bashpid() {
47			# Running bashpid plainly will return incorrect results. This func must
48			# be run in a subshell of the current subshell to get the right pid.
49			# i.e. This will show the wrong value:
50			# bashpid
51			# But this will show the right value:
52			# (bashpid)
53			sh -c 'echo ${PPID}'
54			}
55
56	vapier	1.1	# @FUNCTION: makeopts_jobs
57			# @USAGE: [${MAKEOPTS}]
58			# @DESCRIPTION:
59			# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
60			# specified therein. Useful for running non-make tools in parallel too.
61			# i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
62			# number as bash normalizes it to [0, 255]. If the flags haven't specified a
63			# -j flag, then "1" is shown as that is the default `make` uses. Since there's
64			# no way to represent infinity, we return 999 if the user has -j without a number.
65			makeopts_jobs() {
66			[[ $# -eq 0 ]] && set -- ${MAKEOPTS}
67			# This assumes the first .* will be more greedy than the second .*
68			# since POSIX doesn't specify a non-greedy match (i.e. ".*?").
69			local jobs=$(echo " $* " \| sed -r -n \
70			-e 's:.[[:space:]](-j\|--jobs[=[:space:]])[[:space:]]([0-9]+).*:\2:p' \
71			-e 's:.[[:space:]](-j\|--jobs)[[:space:]].:999:p')
72			echo ${jobs:-1}
73			}
74
75	vapier	1.4	# @FUNCTION: makeopts_loadavg
76			# @USAGE: [${MAKEOPTS}]
77			# @DESCRIPTION:
78			# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the value set
79			# for load-average. For make and ninja based builds this will mean new jobs are
80			# not only limited by the jobs-value, but also by the current load - which might
81			# get excessive due to I/O and not just due to CPU load.
82			# Be aware that the returned number might be a floating-point number. Test
83			# whether your software supports that.
84			makeopts_loadavg() {
85			[[ $# -eq 0 ]] && set -- ${MAKEOPTS}
86			# This assumes the first .* will be more greedy than the second .*
87			# since POSIX doesn't specify a non-greedy match (i.e. ".*?").
88			local lavg=$(echo " $* " \| sed -r -n \
89			-e 's:.[[:space:]](-l\|--load-average[=[:space:]])[[:space:]]([0-9]+\|[0-9]+\.[0-9]+)[^0-9.]*:\2:p' \
90			-e 's:.[[:space:]](-l\|--load-average)[[:space:]].:999:p')
91			echo ${lavg:-1}
92			}
93
94	vapier	1.1	# @FUNCTION: multijob_init
95			# @USAGE: [${MAKEOPTS}]
96			# @DESCRIPTION:
97			# Setup the environment for executing code in parallel.
98			# You must call this before any other multijob function.
99			multijob_init() {
100			# When something goes wrong, try to wait for all the children so we
101			# don't leave any zombies around.
102	vapier	1.2	has wait ${EBUILD_DEATH_HOOKS} \|\| EBUILD_DEATH_HOOKS+=" wait "
103	vapier	1.1
104			# Setup a pipe for children to write their pids to when they finish.
105	vapier	1.3	# We have to allocate two fd's because POSIX has undefined behavior
106			# when you open a FIFO for simultaneous read/write. #487056
107	vapier	1.1	local pipe="${T}/multijob.pipe"
108	vapier	1.3	mkfifo -m 600 "${pipe}"
109			redirect_alloc_fd mj_write_fd "${pipe}"
110			redirect_alloc_fd mj_read_fd "${pipe}"
111	vapier	1.1	rm -f "${pipe}"
112
113			# See how many children we can fork based on the user's settings.
114			mj_max_jobs=$(makeopts_jobs "$@")
115			mj_num_jobs=0
116			}
117
118			# @FUNCTION: multijob_child_init
119			# @USAGE: [--pre\|--post] [command to run in background]
120			# @DESCRIPTION:
121			# This function has two forms. You can use it to execute a simple command
122			# in the background (and it takes care of everything else), or you must
123			# call this first thing in your forked child process.
124			#
125			# The --pre/--post options allow you to select the child generation mode.
126			#
127			# @CODE
128			# # 1st form: pass the command line as arguments:
129			# multijob_child_init ls /dev
130			# # Or if you want to use pre/post fork modes:
131			# multijob_child_init --pre ls /dev
132			# multijob_child_init --post ls /dev
133			#
134			# # 2nd form: execute multiple stuff in the background (post fork):
135			# (
136			# multijob_child_init
137			# out=`ls`
138			# if echo "${out}" \| grep foo ; then
139			# echo "YEAH"
140			# fi
141			# ) &
142			# multijob_post_fork
143			#
144			# # 2nd form: execute multiple stuff in the background (pre fork):
145			# multijob_pre_fork
146			# (
147			# multijob_child_init
148			# out=`ls`
149			# if echo "${out}" \| grep foo ; then
150			# echo "YEAH"
151			# fi
152			# ) &
153			# @CODE
154			multijob_child_init() {
155			local mode="pre"
156			case $1 in
157			--pre) mode="pre" ; shift ;;
158			--post) mode="post"; shift ;;
159			esac
160
161			if [[ $# -eq 0 ]] ; then
162	vapier	1.8	trap 'echo ${BASHPID:-$(bashpid)} $? >&'${mj_write_fd} EXIT
163	vapier	1.1	trap 'exit 1' INT TERM
164			else
165			local ret
166			[[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; }
167			( multijob_child_init ; "$@" ) &
168			[[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; }
169			return ${ret}
170			fi
171			}
172
173			# @FUNCTION: _multijob_fork
174			# @INTERNAL
175			# @DESCRIPTION:
176			# Do the actual book keeping.
177			_multijob_fork() {
178			[[ $# -eq 1 ]] \|\| die "incorrect number of arguments"
179
180			local ret=0
181			[[ $1 == "post" ]] && : $(( ++mj_num_jobs ))
182			if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then
183			multijob_finish_one
184			ret=$?
185			fi
186			[[ $1 == "pre" ]] && : $(( ++mj_num_jobs ))
187			return ${ret}
188			}
189
190			# @FUNCTION: multijob_pre_fork
191			# @DESCRIPTION:
192			# You must call this in the parent process before forking a child process.
193			# If the parallel limit has been hit, it will wait for one child to finish
194			# and return its exit status.
195			multijob_pre_fork() { _multijob_fork pre "$@" ; }
196
197			# @FUNCTION: multijob_post_fork
198			# @DESCRIPTION:
199			# You must call this in the parent process after forking a child process.
200			# If the parallel limit has been hit, it will wait for one child to finish
201			# and return its exit status.
202			multijob_post_fork() { _multijob_fork post "$@" ; }
203
204			# @FUNCTION: multijob_finish_one
205			# @DESCRIPTION:
206			# Wait for a single process to exit and return its exit code.
207			multijob_finish_one() {
208			[[ $# -eq 0 ]] \|\| die "${FUNCNAME} takes no arguments"
209
210			local pid ret
211	vapier	1.3	read -r -u ${mj_read_fd} pid ret \|\| die
212	vapier	1.1	: $(( --mj_num_jobs ))
213			return ${ret}
214			}
215
216			# @FUNCTION: multijob_finish
217			# @DESCRIPTION:
218			# Wait for all pending processes to exit and return the bitwise or
219			# of all their exit codes.
220			multijob_finish() {
221			local ret=0
222			while [[ ${mj_num_jobs} -gt 0 ]] ; do
223			multijob_finish_one
224			: $(( ret \|= $? ))
225			done
226			# Let bash clean up its internal child tracking state.
227			wait
228
229			# Do this after reaping all the children.
230			[[ $# -eq 0 ]] \|\| die "${FUNCNAME} takes no arguments"
231
232	vapier	1.2	# No need to hook anymore.
233			EBUILD_DEATH_HOOKS=${EBUILD_DEATH_HOOKS/ wait / }
234
235	vapier	1.1	return ${ret}
236			}
237
238			# @FUNCTION: redirect_alloc_fd
239			# @USAGE: <var> <file> [redirection]
240			# @DESCRIPTION:
241			# Find a free fd and redirect the specified file via it. Store the new
242			# fd in the specified variable. Useful for the cases where we don't care
243			# about the exact fd #.
244			redirect_alloc_fd() {
245			local var=$1 file=$2 redir=${3:-"<>"}
246
247	jcallen	1.6	# Make sure /dev/fd is sane on Linux hosts. #479656
248	vapier	1.7	if [[ ! -L /dev/fd && ${CBUILD} == linux ]] ; then
249	vapier	1.5	eerror "You're missing a /dev/fd symlink to /proc/self/fd."
250			eerror "Please fix the symlink and check your boot scripts (udev/etc...)."
251			die "/dev/fd is broken"
252			fi
253
254	vapier	1.1	if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then
255			# Newer bash provides this functionality.
256			eval "exec {${var}}${redir}'${file}'"
257			else
258			# Need to provide the functionality ourselves.
259			local fd=10
260			while :; do
261			# Make sure the fd isn't open. It could be a char device,
262			# or a symlink (possibly broken) to something else.
263			if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then
264			eval "exec ${fd}${redir}'${file}'" && break
265			fi
266			[[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?'
267			: $(( ++fd ))
268			done
269			: $(( ${var} = fd ))
270			fi
271			}
272
273			fi