aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBrian Dolbec <dolsen@gentoo.org>2015-11-12 11:26:42 -0800
committerBrian Dolbec <dolsen@gentoo.org>2015-11-12 11:26:42 -0800
commit0cf294d33a466f4a3c537aa82d385ec7f87c7bf3 (patch)
tree6b3ddf4cd221f75fdb84d5bfff03c90aed41cc56
parentBump version, update RELEASE-NOTES (diff)
parentegencache: Delay updating Manifests until all other tasks complete (diff)
downloadportage-0cf294d33a466f4a3c537aa82d385ec7f87c7bf3.tar.gz
portage-0cf294d33a466f4a3c537aa82d385ec7f87c7bf3.tar.bz2
portage-0cf294d33a466f4a3c537aa82d385ec7f87c7bf3.zip
Merge branch 'master' of git+ssh://git.gentoo.org/proj/portage
-rwxr-xr-xbin/egencache70
-rw-r--r--bin/install-qa-check.d/60openrc2
-rw-r--r--bin/save-ebuild-env.sh2
-rw-r--r--man/portage.57
-rwxr-xr-xmisc/emerge-delta-webrsync2
-rw-r--r--pym/portage/emaint/modules/sync/sync.py57
-rw-r--r--pym/portage/repository/config.py6
-rw-r--r--pym/portage/sync/controller.py18
-rw-r--r--pym/portage/sync/modules/git/git.py11
-rw-r--r--pym/portage/sync/modules/rsync/rsync.py17
-rw-r--r--pym/portage/util/_async/AsyncFunction.py5
11 files changed, 139 insertions, 58 deletions
diff --git a/bin/egencache b/bin/egencache
index 51d115ade..a9d4657b9 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -55,7 +55,9 @@ from portage.const import TIMESTAMP_FORMAT
from portage.manifest import guessManifestFileType
from portage.package.ebuild._parallel_manifest.ManifestScheduler import ManifestScheduler
from portage.util import cmp_sort_key, writemsg_level
+from portage.util._async.AsyncFunction import AsyncFunction
from portage.util._async.run_main_scheduler import run_main_scheduler
+from portage.util._async.TaskScheduler import TaskScheduler
from portage.util._eventloop.global_event_loop import global_event_loop
from portage import cpv_getkey
from portage.dep import Atom, isjustname
@@ -748,7 +750,8 @@ class _special_filename(_filename_base):
return self.file_name < other.file_name
class GenChangeLogs(object):
- def __init__(self, portdb, changelog_output, changelog_reversed):
+ def __init__(self, portdb, changelog_output, changelog_reversed,
+ max_jobs=None, max_load=None):
self.returncode = os.EX_OK
self._portdb = portdb
self._wrapper = textwrap.TextWrapper(
@@ -758,6 +761,8 @@ class GenChangeLogs(object):
)
self._changelog_output = changelog_output
self._changelog_reversed = changelog_reversed
+ self._max_jobs = max_jobs
+ self._max_load = max_load
@staticmethod
def grab(cmd):
@@ -882,7 +887,7 @@ class GenChangeLogs(object):
output.close()
- def run(self):
+ def _task_iter(self):
repo_path = self._portdb.porttrees[0]
os.chdir(repo_path)
@@ -908,7 +913,12 @@ class GenChangeLogs(object):
cmod = 0
if float(cmod) < float(lmod):
- self.generate_changelog(cp)
+ yield AsyncFunction(target=self.generate_changelog, args=[cp])
+
+ def run(self):
+ return run_main_scheduler(
+ TaskScheduler(self._task_iter(), event_loop=global_event_loop(),
+ max_jobs=self._max_jobs, max_load=self._max_load))
def egencache_main(args):
@@ -1095,29 +1105,6 @@ def egencache_main(args):
else:
ret.append(gen_cache.returncode)
- if options.update_manifests:
-
- cp_iter = None
- if atoms:
- cp_iter = iter(atoms)
-
- event_loop = global_event_loop()
- scheduler = ManifestScheduler(portdb, cp_iter=cp_iter,
- gpg_cmd=gpg_cmd, gpg_vars=gpg_vars,
- force_sign_key=force_sign_key,
- max_jobs=options.jobs,
- max_load=options.load_average,
- event_loop=event_loop)
-
- signum = run_main_scheduler(scheduler)
- if signum is not None:
- sys.exit(128 + signum)
-
- if options.tolerant:
- ret.append(os.EX_OK)
- else:
- ret.append(scheduler.returncode)
-
if options.update_pkg_desc_index:
if repo_config.writable:
writable_location = repo_config.location
@@ -1149,10 +1136,37 @@ def egencache_main(args):
if options.update_changelogs:
gen_clogs = GenChangeLogs(portdb,
changelog_output=options.changelog_output,
- changelog_reversed=options.changelog_reversed)
- gen_clogs.run()
+ changelog_reversed=options.changelog_reversed,
+ max_jobs=options.jobs,
+ max_load=options.load_average)
+ signum = gen_clogs.run()
+ if signum is not None:
+ sys.exit(128 + signum)
ret.append(gen_clogs.returncode)
+ if options.update_manifests:
+
+ cp_iter = None
+ if atoms:
+ cp_iter = iter(atoms)
+
+ event_loop = global_event_loop()
+ scheduler = ManifestScheduler(portdb, cp_iter=cp_iter,
+ gpg_cmd=gpg_cmd, gpg_vars=gpg_vars,
+ force_sign_key=force_sign_key,
+ max_jobs=options.jobs,
+ max_load=options.load_average,
+ event_loop=event_loop)
+
+ signum = run_main_scheduler(scheduler)
+ if signum is not None:
+ sys.exit(128 + signum)
+
+ if options.tolerant:
+ ret.append(os.EX_OK)
+ else:
+ ret.append(scheduler.returncode)
+
if options.write_timestamp:
timestamp_path = os.path.join(repo_path, 'metadata', 'timestamp.chk')
try:
diff --git a/bin/install-qa-check.d/60openrc b/bin/install-qa-check.d/60openrc
index 9b7fc6d99..1e56b2f37 100644
--- a/bin/install-qa-check.d/60openrc
+++ b/bin/install-qa-check.d/60openrc
@@ -24,7 +24,7 @@ openrc_check() {
for i in "${ED}${d}"/* ; do
[[ -e ${i} ]] || continue
[[ -L ${i} ]] && continue
- f=$("${checkbashisms}" -f "${i}" 2>&1)
+ f=$("${checkbashisms}" -n -f "${i}" 2>&1)
[[ $? != 0 && -n ${f} ]] || continue
eqawarn "QA Notice: shell script appears to use non-POSIX feature(s):"
while read -r ;
diff --git a/bin/save-ebuild-env.sh b/bin/save-ebuild-env.sh
index 477ed28ed..17e4cdcbb 100644
--- a/bin/save-ebuild-env.sh
+++ b/bin/save-ebuild-env.sh
@@ -51,7 +51,7 @@ __save_ebuild_env() {
__dump_trace die \
__quiet_mode __vecho __elog_base eqawarn elog \
einfo einfon ewarn eerror ebegin __eend eend KV_major \
- KV_minor KV_micro KV_to_int get_KV __1 __1 has \
+ KV_minor KV_micro KV_to_int get_KV has \
__has_phase_defined_up_to \
hasv hasq __qa_source __qa_call \
addread addwrite adddeny addpredict __sb_append_var \
diff --git a/man/portage.5 b/man/portage.5
index 8e2be4f63..c9e70a0e3 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1,4 +1,4 @@
-.TH "PORTAGE" "5" "Feb 2015" "Portage VERSION" "Portage"
+.TH "PORTAGE" "5" "Nov 2015" "Portage VERSION" "Portage"
.SH NAME
portage \- the heart of Gentoo
.SH "DESCRIPTION"
@@ -968,6 +968,11 @@ Specifies CVS repository.
Specifies clone depth to use for DVCS repositories. Defaults to 1 (only
the newest commit). If set to 0, the depth is unlimited.
.TP
+.B sync\-hooks\-only\-on\-change
+If set to true, then sync of a given repository will not trigger postsync
+hooks unless hooks would have executed for a master repository or the
+repository has changed since the previous sync operation.
+.TP
.B sync\-type
Specifies type of synchronization performed by `emerge \-\-sync`.
.br
diff --git a/misc/emerge-delta-webrsync b/misc/emerge-delta-webrsync
index 05a0ac721..f2dc8220c 100755
--- a/misc/emerge-delta-webrsync
+++ b/misc/emerge-delta-webrsync
@@ -754,7 +754,7 @@ if [ "$verified" == "1" ]; then
rm -f "${TMPDIR}/portage-${final_date}.tar"
else
echo "recompressing."
- bzip2 -v9 "${TMPDIR}/portage-${final_date}.tar.bz2"
+ bzip2 -v9 "${TMPDIR}/portage-${final_date}.tar"
fi
echo "verifying generated tarball"
diff --git a/pym/portage/emaint/modules/sync/sync.py b/pym/portage/emaint/modules/sync/sync.py
index 57c779d31..15d63e2c8 100644
--- a/pym/portage/emaint/modules/sync/sync.py
+++ b/pym/portage/emaint/modules/sync/sync.py
@@ -233,15 +233,17 @@ class SyncRepos(object):
retvals = sync_scheduler.retvals
msgs.extend(sync_scheduler.msgs)
- # run the post_sync_hook one last time for
- # run only at sync completion hooks
- rcode = sync_manager.perform_post_sync_hook('')
if retvals:
msgs.extend(self.rmessage(retvals, 'sync'))
else:
msgs.extend(self.rmessage([('None', os.EX_OK)], 'sync'))
- if rcode:
- msgs.extend(self.rmessage([('None', rcode)], 'post-sync'))
+
+ # run the post_sync_hook one last time for
+ # run only at sync completion hooks
+ if sync_scheduler.global_hooks_enabled:
+ rcode = sync_manager.perform_post_sync_hook('')
+ if rcode:
+ msgs.extend(self.rmessage([('None', rcode)], 'post-sync'))
# Reload the whole config.
portage._sync_mode = False
@@ -339,6 +341,8 @@ class SyncScheduler(AsyncScheduler):
if master.name in selected_repo_names:
self._repo_map[master.name] = master
self._sync_graph.add(master.name, repo.name)
+ self._complete_graph = self._sync_graph.copy()
+ self._hooks_repos = set()
self._update_leaf_nodes()
def _task_exit(self, task):
@@ -347,9 +351,13 @@ class SyncScheduler(AsyncScheduler):
more leaf nodes.
'''
self._running_tasks.discard(task)
+ # Set hooks_enabled = True by default, in order to ensure
+ # that hooks will be called in a backward-compatible manner
+ # even if all sync tasks have failed.
+ hooks_enabled = True
returncode = task.returncode
if task.returncode == os.EX_OK:
- returncode, message, updatecache_flg = task.result
+ returncode, message, updatecache_flg, hooks_enabled = task.result
if message:
self.msgs.append(message)
repo = task.kwargs['repo'].name
@@ -357,8 +365,38 @@ class SyncScheduler(AsyncScheduler):
self.retvals.append((repo, returncode))
self._sync_graph.remove(repo)
self._update_leaf_nodes()
+ if hooks_enabled:
+ self._hooks_repos.add(repo)
super(SyncScheduler, self)._task_exit(self)
+ def _master_hooks(self, repo_name):
+ """
+ @param repo_name: a repo name
+ @type repo_name: str
+ @return: True if hooks would have been executed for any master
+ repositories of the given repo, False otherwise
+ @rtype: bool
+ """
+ traversed_nodes = set()
+ node_stack = [repo_name]
+ while node_stack:
+ node = node_stack.pop()
+ if node in self._hooks_repos:
+ return True
+ if node not in traversed_nodes:
+ traversed_nodes.add(node)
+ node_stack.extend(self._complete_graph.child_nodes(node))
+ return False
+
+ @property
+ def global_hooks_enabled(self):
+ """
+ @return: True if repo.postsync.d hooks would have been executed
+ for any repositories.
+ @rtype: bool
+ """
+ return bool(self._hooks_repos)
+
def _update_leaf_nodes(self):
'''
Populate self._leaf_nodes with current leaves from
@@ -389,9 +427,10 @@ class SyncScheduler(AsyncScheduler):
self._running_repos.add(node)
self._update_leaf_nodes()
- task = self._sync_manager.async(
- self._emerge_config, self._repo_map[node])
- return task
+ return self._sync_manager.async(
+ emerge_config=self._emerge_config,
+ repo=self._repo_map[node],
+ master_hooks=self._master_hooks(node))
def _can_add_job(self):
'''
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 1060bc7c9..ffb454445 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -87,7 +87,7 @@ class RepoConfig(object):
'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority',
'profile_formats', 'sign_commit', 'sign_manifest',
- 'sync_depth',
+ 'sync_depth', 'sync_hooks_only_on_change',
'sync_type', 'sync_umask', 'sync_uri', 'sync_user', 'thin_manifest',
'update_changelog', 'user_location', '_eapis_banned',
'_eapis_deprecated', '_masters_orig', 'module_specific_options',
@@ -175,6 +175,8 @@ class RepoConfig(object):
self.auto_sync = auto_sync
self.sync_depth = repo_opts.get('sync-depth')
+ self.sync_hooks_only_on_change = repo_opts.get(
+ 'sync-hooks-only-on-change', 'false').lower() == 'true'
self.module_specific_options = {}
@@ -506,7 +508,7 @@ class RepoConfigLoader(object):
# repos.conf is allowed to override.
for k in ('aliases', 'auto_sync', 'eclass_overrides',
'force', 'masters', 'priority',
- 'sync_depth',
+ 'sync_depth', 'sync_hooks_only_on_change',
'sync_type', 'sync_umask', 'sync_uri', 'sync_user',
'module_specific_options'):
v = getattr(repos_conf_opts, k, None)
diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e8132c2ba..4595293c6 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -114,16 +114,17 @@ class SyncManager(object):
return desc
return []
- def async(self, emerge_config=None, repo=None):
+ def async(self, emerge_config=None, repo=None, master_hooks=True):
self.emerge_config = emerge_config
self.settings, self.trees, self.mtimedb = emerge_config
self.xterm_titles = "notitles" not in self.settings.features
self.portdb = self.trees[self.settings['EROOT']]['porttree'].dbapi
return SyncRepo(sync_task=AsyncFunction(target=self.sync,
- kwargs=dict(emerge_config=emerge_config, repo=repo)),
+ kwargs=dict(emerge_config=emerge_config, repo=repo,
+ master_hooks=master_hooks)),
sync_callback=self._sync_callback)
- def sync(self, emerge_config=None, repo=None):
+ def sync(self, emerge_config=None, repo=None, master_hooks=True):
self.callback = None
self.repo = repo
self.exitcode = 1
@@ -156,9 +157,14 @@ class SyncManager(object):
taskmaster = TaskHandler(callback=self.do_callback)
taskmaster.run_tasks(tasks, func, status, options=task_opts)
- self.perform_post_sync_hook(repo.name, repo.sync_uri, repo.location)
+ hooks_enabled = False
+ if (master_hooks or self.updatecache_flg or
+ not repo.sync_hooks_only_on_change):
+ hooks_enabled = True
+ self.perform_post_sync_hook(
+ repo.name, repo.sync_uri, repo.location)
- return self.exitcode, None, self.updatecache_flg
+ return self.exitcode, None, self.updatecache_flg, hooks_enabled
def do_callback(self, result):
@@ -328,7 +334,7 @@ class SyncManager(object):
exitcode = proc.returncode
updatecache_flg = False
if proc.returncode == os.EX_OK:
- exitcode, message, updatecache_flg = proc.result
+ exitcode, message, updatecache_flg, hooks_enabled = proc.result
if updatecache_flg and "metadata-transfer" not in self.settings.features:
updatecache_flg = False
diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py
index c14782c38..179c0dea8 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import logging
+import subprocess
import portage
from portage import os
@@ -81,6 +82,10 @@ class GitSync(NewBase):
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
+ rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
+ previous_rev = subprocess.check_output(rev_cmd,
+ cwd=portage._unicode_encode(self.repo.location))
+
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
@@ -89,4 +94,8 @@ class GitSync(NewBase):
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
- return (os.EX_OK, True)
+
+ current_rev = subprocess.check_output(rev_cmd,
+ cwd=portage._unicode_encode(self.repo.location))
+
+ return (os.EX_OK, current_rev != previous_rev)
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 8ae8a5caf..e0f76b331 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -112,10 +112,10 @@ class RsyncSync(NewBase):
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[6:]
- is_synced, exitcode = self._do_rsync(
+ is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
- return (exitcode, exitcode == os.EX_OK)
+ return (exitcode, updatecache_flg)
retries=0
try:
@@ -138,7 +138,7 @@ class RsyncSync(NewBase):
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
- updatecache_flg=True
+ updatecache_flg = False
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
@@ -240,7 +240,8 @@ class RsyncSync(NewBase):
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
- is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+ is_synced, exitcode, updatecache_flg = self._do_rsync(
+ dosyncuri, timestamp, opts)
if is_synced:
break
@@ -251,7 +252,6 @@ class RsyncSync(NewBase):
else:
# over retries
# exit loop
- updatecache_flg=False
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
@@ -382,6 +382,7 @@ class RsyncSync(NewBase):
def _do_rsync(self, syncuri, timestamp, opts):
+ updatecache_flg = False
is_synced = False
if timestamp != 0 and "--quiet" not in opts:
print(">>> Checking server timestamp ...")
@@ -489,7 +490,7 @@ class RsyncSync(NewBase):
print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
print(">>>")
print()
- return is_synced, exitcode
+ return is_synced, exitcode, updatecache_flg
elif (servertimestamp != 0) and (servertimestamp < timestamp):
self.logger(self.xterm_titles,
">>> Server out of date: %s" % syncuri)
@@ -543,6 +544,8 @@ class RsyncSync(NewBase):
os.unlink(self.servertimestampfile)
except OSError:
pass
+ else:
+ updatecache_flg = True
if exitcode in [0,1,3,4,11,14,20,21]:
is_synced = True
@@ -554,4 +557,4 @@ class RsyncSync(NewBase):
# --prune-empty-directories. Retry for a server that supports
# at least rsync protocol version 29 (>=rsync-2.6.4).
pass
- return is_synced, exitcode
+ return is_synced, exitcode, updatecache_flg
diff --git a/pym/portage/util/_async/AsyncFunction.py b/pym/portage/util/_async/AsyncFunction.py
index b6142a214..40f6c5e75 100644
--- a/pym/portage/util/_async/AsyncFunction.py
+++ b/pym/portage/util/_async/AsyncFunction.py
@@ -15,7 +15,10 @@ class AsyncFunction(ForkProcess):
"result" attribute after the forked process has exited.
"""
- __slots__ = ('args', 'kwargs', 'result', 'target',
+ # NOTE: This class overrides the meaning of the SpawnProcess 'args'
+ # attribute, and uses it to hold the positional arguments for the
+ # 'target' function.
+ __slots__ = ('kwargs', 'result', 'target',
'_async_func_reader', '_async_func_reader_pw')
def _start(self):