aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBrian Dolbec <dolsen@gentoo.org>2017-07-15 00:06:27 +0000
committerZac Medico <zmedico@gentoo.org>2018-03-29 20:51:15 -0700
commit36fd0a505fe03058b64c4e4af331d889426f6a27 (patch)
tree0d1994fd54cdd524b9ed3fc2e4088d800797eb46
parentrepoman: Move errors data to repo metadata/repoman/linechecks.yaml (diff)
downloadportage-36fd0a505fe03058b64c4e4af331d889426f6a27.tar.gz
portage-36fd0a505fe03058b64c4e4af331d889426f6a27.tar.bz2
portage-36fd0a505fe03058b64c4e4af331d889426f6a27.zip
repoman: Add a new config.py file with config loading utilities
These include recursively merging of multiple yaml files. They are needed for masters stacking.
-rw-r--r--repoman/pym/repoman/config.py151
1 files changed, 151 insertions, 0 deletions
diff --git a/repoman/pym/repoman/config.py b/repoman/pym/repoman/config.py
new file mode 100644
index 000000000..9da30ed9e
--- /dev/null
+++ b/repoman/pym/repoman/config.py
@@ -0,0 +1,151 @@
+# -*- coding:utf-8 -*-
+
+import copy
+import itertools
+import json
+import os
+import stat
+
+import yaml
+
+
+class ConfigError(Exception):
+ """Raised when a config file fails to load"""
+ pass
+
+
+def merge_config(base, head):
+ """
+ Merge two JSON or YAML documents into a single object. Arrays are
+ merged by extension. If dissimilar types are encountered, then the
+ head value overwrites the base value.
+ """
+
+ if isinstance(head, dict):
+ if not isinstance(base, dict):
+ return copy.deepcopy(head)
+
+ result = {}
+ for k in itertools.chain(head, base):
+ try:
+ result[k] = merge_config(base[k], head[k])
+ except KeyError:
+ try:
+ result[k] = copy.deepcopy(head[k])
+ except KeyError:
+ result[k] = copy.deepcopy(base[k])
+
+ elif isinstance(head, list):
+ result = []
+ if not isinstance(base, list):
+ result.extend(copy.deepcopy(x) for x in head)
+ else:
+ if any(isinstance(x, (dict, list)) for x in itertools.chain(head, base)):
+ # merge items with identical indexes
+ for x, y in zip(base, head):
+ if isinstance(x, (dict, list)):
+ result.append(merge_config(x, y))
+ else:
+ # head overwrites base (preserving index)
+ result.append(copy.deepcopy(y))
+ # copy remaining items from the longer list
+ if len(base) != len(head):
+ if len(base) > len(head):
+ result.extend(copy.deepcopy(x) for x in base[len(head):])
+ else:
+ result.extend(copy.deepcopy(x) for x in head[len(base):])
+ else:
+ result.extend(copy.deepcopy(x) for x in base)
+ result.extend(copy.deepcopy(x) for x in head)
+
+ else:
+ result = copy.deepcopy(head)
+
+ return result
+
+def _yaml_load(filename):
+ """
+ Load filename as YAML and return a dict. Raise ConfigError if
+ it fails to load.
+ """
+ with open(filename, 'rt') as f:
+ try:
+ return yaml.safe_load(f)
+ except yaml.parser.ParserError as e:
+ raise ConfigError("{}: {}".format(filename, e))
+
+def _json_load(filename):
+ """
+ Load filename as JSON and return a dict. Raise ConfigError if
+ it fails to load.
+ """
+ with open(filename, 'rt') as f:
+ try:
+ return json.load(f) #nosec
+ except ValueError as e:
+ raise ConfigError("{}: {}".format(filename, e))
+
+def iter_files(files_dirs):
+ """
+ Iterate over nested file paths in lexical order.
+ """
+ stack = list(reversed(files_dirs))
+ while stack:
+ location = stack.pop()
+ try:
+ st = os.stat(location)
+ except FileNotFoundError:
+ continue
+
+ if stat.S_ISDIR(st.st_mode):
+ stack.extend(os.path.join(location, x)
+ for x in sorted(os.listdir(location), reverse=True))
+
+ elif stat.S_ISREG(st.st_mode):
+ yield location
+
+def load_config(conf_dirs, file_extensions=None, valid_versions=None):
+ """
+ Load JSON and/or YAML files from a directories, and merge them together
+ into a single object.
+
+ @param conf_dirs: ordered iterable of directories to load the config from
+ @param file_extensions: Optional list of file extension types to load
+ @param valid_versions: list of compatible file versions allowed
+ @returns: the stacked config
+ """
+
+ result = {}
+ for filename in iter_files(conf_dirs):
+ if file_extensions is not None and not filename.endswith(file_extensions):
+ continue
+
+ loaders = []
+ extension = filename.rsplit('.', 1)[1]
+ if extension in ['json']:
+ loaders.append(_json_load)
+ elif extension in ['yml', 'yaml']:
+ loaders.append(_yaml_load)
+
+ config = None
+ exception = None
+ for loader in loaders:
+ try:
+ config = loader(filename) or {}
+ except ConfigError as e:
+ exception = e
+ else:
+ break
+
+ if config is None:
+ print("Repoman.config.load_config(), Error loading file: %s" % filename)
+ print(" Aborting...")
+ raise exception
+
+ if config:
+ if config['version'] not in valid_versions:
+ raise ConfigError("Invalid file version: %s in: %s\nPlease upgrade repoman: current valid versions: %s"
+ % (config['version'], filename, valid_versions))
+ result = merge_config(result, config)
+
+ return result