aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBrian Dolbec <dolsen@gentoo.org>2017-07-10 17:39:04 +0000
committerZac Medico <zmedico@gentoo.org>2018-03-29 20:51:14 -0700
commit939cc6a60b01659b36b4b03b6568932d595fb8eb (patch)
treec1a5be46fbd7d75fb0651891df4838ecb0b26a19
parentrepoman: modules/scan/*: Add module_runsIn spec variable (diff)
downloadportage-939cc6a60b01659b36b4b03b6568932d595fb8eb.tar.gz
portage-939cc6a60b01659b36b4b03b6568932d595fb8eb.tar.bz2
portage-939cc6a60b01659b36b4b03b6568932d595fb8eb.zip
repoman: Create a new ModuleConfig class
This class will load the module configuration data from the target repo. Move the Modules class handling code to this new class. Update scanner.py to use the new ModuleConfig class for Modules class calls. Move the hard-coded loop lists to use the ModuleConfig instance configure/dynamically determined lists. TODO: add masters inheritance config stacking. Signed-off-by: Brian Dolbec <dolsen@gentoo.org>
-rw-r--r--repoman/pym/repoman/modules/scan/module.py84
-rw-r--r--repoman/pym/repoman/scanner.py83
2 files changed, 118 insertions, 49 deletions
diff --git a/repoman/pym/repoman/modules/scan/module.py b/repoman/pym/repoman/modules/scan/module.py
new file mode 100644
index 000000000..1f9e85408
--- /dev/null
+++ b/repoman/pym/repoman/modules/scan/module.py
@@ -0,0 +1,84 @@
+
+'''
+moudules/scan/module.py
+Module loading and run list generator
+'''
+
+import logging
+import os
+import yaml
+
+from portage.module import InvalidModuleName, Modules
+from portage.util import stack_lists
+
+MODULES_PATH = os.path.dirname(__file__)
+# initial development debug info
+logging.debug("module path: %s", MODULES_PATH)
+
+
+class ModuleConfig(object):
+ '''Holds the scan modules configuration information and
+ creates the ordered list of modulles to run'''
+
+ def __init__(self, configpaths):
+ '''Module init
+
+ @param configpaths: ordered list of filepaths to load
+ '''
+ self.configpaths = configpaths
+
+ self.controller = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
+ logging.debug("module_names: %s", self.controller.module_names)
+
+ self._configs = None
+ self.enabled = []
+ self.pkgs_loop = []
+ self.ebuilds_loop = []
+ self.final_loop = []
+ self.modules_forced = ['ebuild', 'mtime']
+ self.load_configs()
+ for loop in ['pkgs', 'ebuilds', 'final']:
+ logging.debug("ModuleConfig; Processing loop %s", loop)
+ setattr(self, '%s_loop' % loop, self._determine_list(loop))
+
+
+ def load_configs(self, configpaths=None):
+ '''load the config files in order'''
+ if configpaths:
+ self.configpaths = configpaths
+ elif not self.configpaths:
+ logging.error("ModuleConfig; Error: No repository.yml files defined")
+ configs = []
+ for path in self.configpaths:
+ logging.debug("ModuleConfig; Processing: %s", path)
+ if os.path.exists(path):
+ try:
+ with open(path, 'r') as inputfile:
+ configs.append(yaml.safe_load(inputfile.read()))
+ except IOError as error:
+ logging,error("Failed to load file: %s", inputfile)
+ logging.exception(error)
+ logging.debug("ModuleConfig; completed : %s", path)
+ logging.debug("ModuleConfig; new _configs: %s", configs)
+ self._configs = configs
+
+ def _determine_list(self, loop):
+ '''Determine the ordered list from the config data and
+ the moule_runsIn value in the module_spec
+
+ @returns: list of modules
+ '''
+ lists = [c['scan_modules'].split() for c in self._configs]
+ stacked = self.modules_forced + stack_lists(lists)
+ mlist = []
+ try:
+ for mod in stacked:
+ logging.debug("ModuleConfig; checking loop %s, module: %s, in: %s",
+ loop, mod, self.controller.get_spec(mod, 'module_runsIn'))
+ if loop in self.controller.get_spec(mod, 'module_runsIn'):
+ mlist.append(mod)
+ except InvalidModuleName:
+ logging.error("ModuleConfig; unknown module: %s, skipping", mod)
+
+ logging.debug("ModuleConfig; mlist: %s", mlist)
+ return mlist
diff --git a/repoman/pym/repoman/scanner.py b/repoman/pym/repoman/scanner.py
index e13d4f68b..ffce701be 100644
--- a/repoman/pym/repoman/scanner.py
+++ b/repoman/pym/repoman/scanner.py
@@ -15,20 +15,10 @@ from repoman.modules.commit import repochecks
from repoman.modules.commit import manifest
from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
from repoman.repos import repo_metadata
+from repoman.modules.scan.module import ModuleConfig
from repoman.modules.scan.scan import scan
from repoman.modules.vcs.vcs import vcs_files_to_cps
-from portage.module import Modules
-
-MODULES_PATH = os.path.join(os.path.dirname(__file__), "modules", "scan")
-# initial development debug info
-logging.debug("module path: %s", MODULES_PATH)
-
-MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
-
-MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
-# initial development debug info
-logging.debug("module_names: %s", MODULE_NAMES)
DATA_TYPES = {'dict': dict, 'Future': ExtendedFuture, 'list': list, 'set': set}
@@ -124,6 +114,12 @@ class Scanner(object):
if self.vcs_settings.vcs is None:
self.options.echangelog = 'n'
+ # Initialize the ModuleConfig class here
+ # TODO Add layout.conf masters repository.yml config to the list to load/stack
+ self.moduleconfig = ModuleConfig([
+ '/home/bdolbec/git/gentoo/metadata/repoman/repository.yml',
+ ])
+
checks = {}
# The --echangelog option causes automatic ChangeLog generation,
# which invalidates changelog.ebuildadded and changelog.missing
@@ -212,7 +208,7 @@ class Scanner(object):
@returns: dictionary
'''
kwargs = {}
- for key in MODULE_CONTROLLER.modules[mod]['mod_kwargs']:
+ for key in self.moduleconfig.controller.modules[mod]['mod_kwargs']:
kwargs[key] = self.kwargs[key]
return kwargs
@@ -224,7 +220,7 @@ class Scanner(object):
@param dynamic_data: dictionary structure
@returns: dictionary
'''
- func_kwargs = MODULE_CONTROLLER.modules[mod]['func_kwargs']
+ func_kwargs = self.moduleconfig.controller.modules[mod]['func_kwargs']
# determine new keys
required = set(list(func_kwargs))
exist = set(list(dynamic_data))
@@ -324,19 +320,16 @@ class Scanner(object):
'validity_future',
]
# need to set it up for ==> self.modules or some other ordered list
- for mod in [('manifests', 'Manifests'), ('ebuild', 'Ebuild'),
- ('keywords', 'KeywordChecks'), ('files', 'FileChecks'),
- ('fetches', 'FetchChecks'),
- ('pkgmetadata', 'PkgMetadata'),
- ]:
- mod_class = MODULE_CONTROLLER.get_class(mod[0])
+ logging.debug("***** starting pkgs_loop: %s", self.moduleconfig.pkgs_loop)
+ for mod in self.moduleconfig.pkgs_loop:
+ mod_class = self.moduleconfig.controller.get_class(mod)
logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod[0]))
- logging.debug("scan_pkgs; module: %s", mod[1])
- do_it, functions = self.modules[mod[1]].runInPkgs
+ self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
+ logging.debug("scan_pkgs; module: %s", mod_class.__name__)
+ do_it, functions = self.modules[mod_class.__name__].runInPkgs
if do_it:
for func in functions:
- _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod, dynamic_data))
if _continue:
# If we can't access all the metadata then it's totally unsafe to
# commit since there's no way to generate a correct Manifest.
@@ -373,28 +366,19 @@ class Scanner(object):
# initialize per ebuild plugin checks here
# need to set it up for ==> self.modules_list or some other ordered list
- for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
- ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'),
- ('fetches', 'FetchChecks'),
- ('description', 'DescriptionChecks'),
- ('keywords', 'KeywordChecks'),
- ('pkgmetadata', 'PkgMetadata'), ('ruby', 'RubyEclassChecks'),
- ('restrict', 'RestrictChecks'),
- ('mtime', 'MtimeChecks'), ('multicheck', 'MultiCheck'),
- # Options.is_forced() is used to bypass further checks
- ('options', 'Options'), ('profile', 'ProfileDependsChecks'),
- ]:
- if mod[0] and mod[1] not in self.modules:
- mod_class = MODULE_CONTROLLER.get_class(mod[0])
- logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
- logging.debug("scan_ebuilds: module: %s", mod[1])
- do_it, functions = self.modules[mod[1]].runInEbuilds
+ for mod in self.moduleconfig.ebuilds_loop:
+ if mod:
+ mod_class = self.moduleconfig.controller.get_class(mod)
+ if mod_class.__name__ not in self.modules:
+ logging.debug("Initializing class name: %s", mod_class.__name__)
+ self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
+ logging.debug("scan_ebuilds: module: %s", mod_class.__name__)
+ do_it, functions = self.modules[mod_class.__name__].runInEbuilds
logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning function: %s", func)
- _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod, dynamic_data))
if _continue:
# If we can't access all the metadata then it's totally unsafe to
# commit since there's no way to generate a correct Manifest.
@@ -414,18 +398,19 @@ class Scanner(object):
# initialize per pkg plugin final checks here
# need to set it up for ==> self.modules_list or some other ordered list
xpkg_complete = False
- for mod in [('pkgmetadata', 'PkgMetadata'), ('keywords', 'KeywordChecks')]:
- if mod[0] and mod[1] not in self.modules:
- mod_class = MODULE_CONTROLLER.get_class(mod[0])
- logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
- logging.debug("scan_ebuilds final checks: module: %s", mod[1])
- do_it, functions = self.modules[mod[1]].runInFinal
+ for mod in self.moduleconfig.final_loop:
+ if mod:
+ mod_class = self.moduleconfig.controller.get_class(mod)
+ if mod_class.__name__ not in self.modules:
+ logging.debug("Initializing class name: %s", mod_class.__name__)
+ self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
+ logging.debug("scan_ebuilds final checks: module: %s", mod_class.__name__)
+ do_it, functions = self.modules[mod_class.__name__].runInFinal
logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning function: %s", func)
- _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod, dynamic_data))
if _continue:
xpkg_complete = True
# logging.debug("\t>>> Continuing")