qs/src/GLIPortage.py

"""
# Copyright 1999-2005 Gentoo Foundation
# This source code is distributed under the terms of version 2 of the GNU
# General Public License as published by the Free Software Foundation, a copy
# of which can be found in the main directory of this project.
Gentoo Linux Installer

$Id: GLIPortage.py,v 1.55 2006/09/13 16:50:23 agaffney Exp $
"""

import re
import os
import sys
import GLIUtility
from GLIException import GLIException

class GLIPortage(object):

        def __init__(self, chroot_dir, grp_install, logger, debug, cc, compile_logfile):
                self._chroot_dir = chroot_dir
                self._grp_install = grp_install
                self._logger = logger
                self._debug = debug
                self._cc = cc
                self._compile_logfile = compile_logfile

        def get_deps(self, pkgs):
                pkglist = []
                if isinstance(pkgs, str):
                        pkgs = pkgs.split()
                for pkg in pkgs:
                        if not pkg: continue
                        if self._debug: self._logger.log("get_deps(): pkg is " + pkg)
                        if not self._grp_install or not self.get_best_version_vdb(pkg):
                                if self._debug: self._logger.log("get_deps(): grabbing compile deps")
                                tmppkglist = GLIUtility.spawn("emerge -p " + pkg + r" 2>/dev/null | grep -e '^\[[a-z]' | cut -d ']' -f2 | sed -e 's:^ ::' -e 's: .\+$::'", chroot=self._chroot_dir, return_output=True)[1].strip().split("\n")
                        else:
                                if self._debug: self._logger.log("get_deps(): grabbing binary deps")
                                # The runtimedeps.py script generates a package install order that is *very* different from emerge itself
#                               tmppkglist = GLIUtility.spawn("python ../../runtimedeps.py " + self._chroot_dir + " " + pkg, return_output=True)[1].strip().split("\n")
                                tmppkglist = []
                                for tmppkg in GLIUtility.spawn("emerge -p " + pkg + r" 2>/dev/null | grep -e '^\[[a-z]' | cut -d ']' -f2 | sed -e 's:^ ::' -e 's: .\+$::'", chroot=self._chroot_dir, return_output=True)[1].strip().split("\n"):
                                        if self._debug: self._logger.log("get_deps(): looking at " + tmppkg)
                                        if self.get_best_version_vdb("=" + tmppkg):
                                                if self._debug: self._logger.log("get_deps(): package " + tmppkg + " in host vdb...adding to tmppkglist")
                                                tmppkglist.append(tmppkg)
                        if self._debug: self._logger.log("get_deps(): deplist for " + pkg + ": " + str(tmppkglist))
                        for tmppkg in tmppkglist:
                                if self._debug: self._logger.log("get_deps(): checking to see if " + tmppkg + " is already in pkglist")
                                if not tmppkg in pkglist and not self.get_best_version_vdb_chroot("=" + tmppkg):
                                        if self._debug: self._logger.log("get_deps(): adding " + tmppkg + " to pkglist")
                                        pkglist.append(tmppkg)
                if self._debug: self._logger.log("get_deps(): pkglist is " + str(pkglist))
                if not pkglist:
                                raise GLIException("GetDepListError", 'fatal', 'get_deps', "Dep list is empty. This usually means there is no portage tree in the chroot")
                return pkglist

        def parse_vdb_contents(self, file):
                entries = []
                try:
                        vdbfile = open(file, "r")
                except:
                        return entries
                for line in vdbfile.readlines():
                        parts = line.strip().split(" ")
                        if parts[0] == "obj":
                                entries.append(parts[1])
#                       elif parts[0] == "dir":
#                               entries.append(parts[1] + "/")
                        elif parts[0] == "sym":
                                entries.append(" ".join(parts[1:4]))
                entries.sort()
                return entries

        def copy_pkg_to_chroot(self, package, use_root=False, ignore_missing=False):
                symlinks = { '/bin': '/mnt/livecd/bin/', '/boot': '/mnt/livecd/boot/', '/lib': '/mnt/livecd/lib/', 
                             '/opt': '/mnt/livecd/opt/', '/sbin': '/mnt/livecd/sbin/', '/usr': '/mnt/livecd/usr/',
                             '/etc/gconf': '/usr/livecd/gconf/' }

                tmpdir = "/var/tmp"
                image_dir = tmpdir + "/portage/" + package.split("/")[1] + "/image"
                root_cmd = ""
                tmp_chroot_dir = self._chroot_dir
                portage_tmpdir = "/var/tmp"
                vdb_dir = "/var/db/pkg/"
                if use_root:
                        root_cmd = "ROOT=" + self._chroot_dir
                        tmp_chroot_dir = ""
                        portage_tmpdir = self._chroot_dir + "/var/tmp"
                        vdb_dir = self._chroot_dir + "/var/db/pkg/"

                # Create /tmp, /var/tmp, and /var/lib/portage with proper permissions
                oldumask = os.umask(0)
                if not os.path.exists(self._chroot_dir + "/tmp"):
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): /tmp doesn't exist in chroot...creating with proper permissions")
                        try:
                                os.mkdir(self._chroot_dir + "/tmp", 01777)
                        except:
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Failed to create /tmp in chroot")
                if not os.path.exists(self._chroot_dir + "/var/tmp"):
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): /var/tmp doesn't exist in chroot...creating with proper permissions")
                        try:
                                os.mkdir(self._chroot_dir + "/var", 0755)
                        except:
                                pass
                        try:
                                os.mkdir(self._chroot_dir + "/var/tmp", 01777)
                        except:
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Failed to create /var/tmp in chroot")
                if not os.path.exists(self._chroot_dir + "/var/lib/portage"):
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): /var/lib/portage doesn't exist in chroot...creating with proper permissions")
                        try:
                                os.mkdir(self._chroot_dir + "/var/lib", 0755)
                        except:
                                pass
                        try:
                                os.mkdir(self._chroot_dir + "/var/lib/portage", 02750)
                        except:
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Failed to create /var/lib/portage in chroot")
                os.umask(oldumask)

                # Check to see if package is actually in vdb
                if not GLIUtility.is_file("/var/db/pkg/" + package):
                        if ignore_missing:
                                if self._debug:
                                        self._logger.log("DEBUG: copy_pkg_to_chroot(): package " + package + " does not have a vdb entry but ignore_missing=True...ignoring error")
                                return
                        else:
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "There is no vdb entry for " + package)

                # Copy the vdb entry for the package from the LiveCD to the chroot
                if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): copying vdb entry for " + package)
                if not GLIUtility.exitsuccess(GLIUtility.spawn("mkdir -p " + self._chroot_dir + "/var/db/pkg/" + package + " && cp -a /var/db/pkg/" + package + "/* " + self._chroot_dir + "/var/db/pkg/" + package, logfile=self._compile_logfile, append_log=True)):
                        raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not copy vdb entry for " + package)

                # Create the image dir in the chroot
                if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running 'mkdir -p " + self._chroot_dir + image_dir + "'")
                if not GLIUtility.exitsuccess(GLIUtility.spawn("mkdir -p " + self._chroot_dir + image_dir, logfile=self._compile_logfile, append_log=True)):
                        raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not create image dir for " + package)

                # Create list of files for tar to work with from CONTENTS file in vdb entry
                entries = self.parse_vdb_contents("/var/db/pkg/" + package + "/CONTENTS")
                if not entries:
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): no files for " + package + "...skipping tar and symlink fixup")
                else:
#                       if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot: files for " + package + ": " + str(entries))
                        try:
                                tarfiles = open("/tmp/tarfilelist", "w")
                                for entry in entries:
                                        parts = entry.split(" ")
#                                       # Hack for symlink crappiness
#                                       for symlink in symlinks:
#                                               if parts[0].startswith(symlink):
#                                                       parts[0] = symlinks[symlink] + parts[0][len(symlink):]
                                        tarfiles.write(parts[0] + "\n")
                                tarfiles.close()
                        except:
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not create filelist for " + package)

                        # Use tar to transfer files into IMAGE directory
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running 'tar -cp --files-from=/tmp/tarfilelist --no-recursion 2>/dev/null | tar -C " + self._chroot_dir + image_dir + " -xp'")
                        if not GLIUtility.exitsuccess(GLIUtility.spawn("tar -cp --files-from=/tmp/tarfilelist --no-recursion 2>/dev/null | tar -C " + self._chroot_dir + image_dir + " -xp", logfile=self._compile_logfile, append_log=True)):
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not execute tar for " + package)

                        # Fix mode, uid, and gid of directories
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running find " + self._chroot_dir + image_dir + " -mindepth 1 -type d 2>/dev/null | sed -e 's:^" + self._chroot_dir + image_dir + "::' | grep -v '^$'")
                        dirlist = GLIUtility.spawn("find " + self._chroot_dir + image_dir + " -mindepth 1 -type d 2>/dev/null | sed -e 's:^" + self._chroot_dir + image_dir + "::' | grep -v '^$'", return_output=True)[1].strip().split("\n")
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): found the following directories: " + str(dirlist))
                        if not dirlist or dirlist[0] == "":
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "directory list entry for " + package + "...this shouldn't happen!")
                        for dir in dirlist:
                                dirstat = os.stat(dir)
                                if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): setting mode " + str(dirstat[0]) + " and uid/gid " + str(dirstat[4]) + "/" + str(dirstat[5]) + " for directory " + self._chroot_dir + image_dir + dir)
                                os.chown(self._chroot_dir + image_dir + dir, dirstat[4], dirstat[5])
                                os.chmod(self._chroot_dir + image_dir + dir, dirstat[0])

#                       # More symlink crappiness hacks
#                       for symlink in symlinks:
##                              if GLIUtility.is_file(self._chroot_dir + image_dir + symlinks[symlink]):
#                               if os.path.islink(self._chroot_dir + image_dir + symlink):
#                                       if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): fixing " + symlink + " symlink ickiness stuff in " + image_dir + " for " + package)
#                                       GLIUtility.spawn("rm " + self._chroot_dir + image_dir + symlink)
#                                       if not GLIUtility.exitsuccess(GLIUtility.spawn("mv " + self._chroot_dir + image_dir + symlinks[symlink] + " " + self._chroot_dir + image_dir + symlink)):
#                                               raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not fix " + symlink + " symlink ickiness for " + package)

                # Run pkg_setup
                if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running pkg_setup for " + package)
                if not GLIUtility.exitsuccess(GLIUtility.spawn("env " + root_cmd + " PORTAGE_TMPDIR=" + portage_tmpdir + " FEATURES=noauto ebuild " + vdb_dir + package + "/*.ebuild setup", chroot=tmp_chroot_dir, logfile=self._compile_logfile, append_log=True)):
                        raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not execute pkg_setup for " + package)

                # Run pkg_preinst
                if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running preinst for " + package)
                if not GLIUtility.exitsuccess(GLIUtility.spawn("env " + root_cmd + " PORTAGE_TMPDIR=" + portage_tmpdir + " FEATURES=noauto ebuild " + vdb_dir + package + "/*.ebuild preinst", chroot=tmp_chroot_dir, logfile=self._compile_logfile, append_log=True)):
                        raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not execute preinst for " + package)

                # Copy files from image_dir to chroot
                if not entries:
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): no files for " + package + "...skipping copy from image dir to /")
                else:
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): copying files from " + image_dir + " to / for " + package)
#                       if not GLIUtility.exitsuccess(GLIUtility.spawn("cp -a " + self._chroot_dir + image_dir + "/* " + self._chroot_dir)):
                        if not GLIUtility.exitsuccess(GLIUtility.spawn("tar -C " + self._chroot_dir + image_dir + "/ -cp . | tar -C " + self._chroot_dir + "/ -xp", logfile=self._compile_logfile, append_log=True)):
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not copy files from " + image_dir + " to / for " + package)

                # Run pkg_postinst
                if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running postinst for " + package)
                if not GLIUtility.exitsuccess(GLIUtility.spawn("env " + root_cmd + " PORTAGE_TMPDIR=" + portage_tmpdir + " FEATURES=noauto ebuild " + vdb_dir + package + "/*.ebuild postinst", chroot=tmp_chroot_dir, logfile=self._compile_logfile, append_log=True)):
                        raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not execute postinst for " + package)

                # Remove image_dir
                if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): removing " + image_dir + " for " + package)
                if not GLIUtility.exitsuccess(GLIUtility.spawn("rm -rf " + self._chroot_dir + image_dir, logfile=self._compile_logfile, append_log=True)):
                        raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not remove + " + image_dir + " for " + package)

                # Run env-update
                if not use_root:
                        if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running env-update inside chroot")
                        if not GLIUtility.exitsuccess(GLIUtility.spawn("env-update", chroot=self._chroot_dir, logfile=self._compile_logfile, append_log=True)):
                                raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not run env-update for " + package)

        def add_pkg_to_world(self, package):
                if package.find("/") == -1:
                        package = self.get_best_version_vdb_chroot(package)
                if not package: return False
                expr = re.compile('^=?(.+?/.+?)(-\d.+)?$')
                res = expr.match(package)
                if res:
                        GLIUtility.spawn("echo " + res.group(1) + " >> " + self._chroot_dir + "/var/lib/portage/world")

        def get_best_version_vdb(self, package):
                if package.startswith('='):
                        package = package[1:]
                        if GLIUtility.is_file("/var/db/pkg/" + package):
                                return package
                        else:
                                return ""
                else:
                        return GLIUtility.spawn("portageq best_version / " + package, return_output=True)[1].strip()

        def get_best_version_vdb_chroot(self, package):
                if package.startswith('='):
                        package = package[1:]
                        if GLIUtility.is_file(self._chroot_dir + "/var/db/pkg/" + package):
                                return package
                        else:
                                return ""
                else:
                        return GLIUtility.spawn("portageq best_version / " + package, chroot=self._chroot_dir, return_output=True)[1].strip()

#       def get_best_version_tree(self, package):
#               return portage.best(tree.match(package))

        def emerge(self, packages, add_to_world=True):
                if isinstance(packages, str):
                        packages = packages.split()
                self._cc.addNotification("progress", (0, "Calculating dependencies for " + " ".join(packages)))
                pkglist = self.get_deps(packages)
                if self._debug: self._logger.log("install_packages(): pkglist is " + str(pkglist))
                for i, pkg in enumerate(pkglist):
                        if not pkg: continue
                        if self._debug: self._logger.log("install_packages(): processing package " + pkg)
                        self._cc.addNotification("progress", (float(i) / len(pkglist), "Emerging " + pkg + " (" + str(i+1) + "/" + str(len(pkglist)) + ")"))
                        if not self._grp_install or not self.get_best_version_vdb("=" + pkg):
                                status = GLIUtility.spawn("emerge -1 =" + pkg, display_on_tty8=True, chroot=self._chroot_dir, logfile=self._compile_logfile, append_log=True)
#                               status = self._emerge("=" + pkg)
                                if not GLIUtility.exitsuccess(status):
                                        raise GLIException("EmergePackageError", "fatal", "emerge", "Could not emerge " + pkg + "!")
                        else:
#                               try:
                                self.copy_pkg_to_chroot(pkg)
#                               except:
#                                       raise GLIException("EmergePackageError", "fatal", "emerge", "Could not emerge " + pkg + "!")
                        self._cc.addNotification("progress", (float(i+1) / len(pkglist), "Done emerging " + pkg + " (" + str(i+1) + "/" + str(len(pkglist)) + ")"))
                if add_to_world:
                        for package in packages:
                                self.add_pkg_to_world(package)


def usage(progname):
        print """
Usage: %s [-c|--chroot-dir <chroot directory>] [-g|--grp] [-s|--stage3] [-h|--help]

Options:

  -c|--chroot-dir   Specifies the directory where your chroot is. This is
                    "/mnt/gentoo" by default.

  -g|--grp          Install specified packages and dependencies into chroot
                    by using files from the LiveCD.

  -s|--stage3       Create a stage3 equivelant in the chroot directory by using
                    files from the LiveCD.

  -h|--help         Display this help
""" % (progname)

if __name__ == "__main__":
        chroot_dir = "/mnt/gentoo"
        mode = None
        grp_packages = []
        progname = sys.argv.pop(0)
        while len(sys.argv):
                arg = sys.argv.pop(0)
                if arg == "-c" or arg == "--chroot-dir":
                        chroot_dir = sys.argv.pop(0)
                elif arg == "-g" or arg == "--grp":
                        mode = "grp"
                elif arg == "-s" or arg == "--stage3":
                        mode = "stage3"
                elif arg == "-h" or arg == "--help":
                        usage(progname)
                        sys.exit(0)
                elif arg[0] == "-":
                        usage(progname)
                        sys.exit(1)
                else:
                        grp_packages.append(arg)

        gliportage = GLIPortage(chroot_dir, True, None, False, None, None)
        if mode == "stage3":
                if not GLIUtility.is_file("/usr/livecd/systempkgs.txt"):
                        print "Required file /usr/livecd/systempkgs.txt does not exist!"
                        sys.exit(1)
                try:
                        syspkgs = open("/usr/livecd/systempkgs.txt", "r")
                        systempkgs = syspkgs.readlines()
                        syspkgs.close()
                except:
                        print "Could not open /usr/livecd/systempkgs.txt!"
                        sys.exit(1)

                # Pre-create /lib (and possible /lib32 and /lib64)
                if os.path.islink("/lib") and os.readlink("/lib") == "lib64":
                        if not GLIUtility.exitsuccess(GLIUtility.spawn("mkdir " + chroot_dir + "/lib64 && ln -s lib64 " + chroot_dir + "/lib")):
                                print "Could not precreate /lib64 dir and /lib -> /lib64 symlink"
                                sys.exit(1)

                syspkglen = len(systempkgs)
                for i, pkg in enumerate(systempkgs):
                        pkg = pkg.strip()
                        print "Copying " + pkg + " (" + str(i+1) + "/" + str(syspkglen) + ")"
                        gliportage.copy_pkg_to_chroot(pkg, True, ignore_missing=True)
                GLIUtility.spawn("cp /etc/make.conf " + chroot_dir + "/etc/make.conf")
                GLIUtility.spawn("ln -s `readlink /etc/make.profile` " + chroot_dir + "/etc/make.profile")
                GLIUtility.spawn("cp -f /etc/inittab.old " + chroot_dir + "/etc/inittab")

                # Nasty, nasty, nasty hack because vapier is a tool
                for tmpfile in ("/etc/passwd", "/etc/group", "/etc/shadow"):
                        GLIUtility.spawn("grep -ve '^gentoo' " + tmpfile + " > " + chroot_dir + tmpfile)

                chrootscript = r"""
                #!/bin/bash

                source /etc/make.conf
                export LDPATH="/usr/lib/gcc-lib/${CHOST}/$(cd /usr/lib/gcc-lib/${CHOST} && ls -1 | head -n 1)"

                ldconfig $LDPATH
                gcc-config 1
                env-update
                source /etc/profile
                modules-update
                [ -f /usr/bin/binutils-config ] && binutils-config 1
                source /etc/profile
                #mount -t proc none /proc
                #cd /dev
                #/sbin/MAKEDEV generic-i386
                #umount /proc
                [ -f /lib/udev-state/devices.tar.bz2 ] && tar -C /dev -xjf /lib/udev-state/devices.tar.bz2
                """
                script = open(chroot_dir + "/tmp/extrastuff.sh", "w")
                script.write(chrootscript)
                script.close()
                GLIUtility.spawn("chmod 755 /tmp/extrastuff.sh && /tmp/extrastuff.sh", chroot=chroot_dir)
                GLIUtility.spawn("rm -rf /var/tmp/portage/* /usr/portage /tmp/*", chroot=chroot_dir)
                print "Stage3 equivelant generation complete!"
        elif mode == "grp":
                for pkg in grp_packages:
                        if not gliportage.get_best_version_vdb(pkg):
                                print "Package " + pkg + " is not available for install from the LiveCD"
                                continue
                        pkglist = gliportage.get_deps(pkg)
                        for i, tmppkg in enumerate(pkglist):
                                print "Copying " + tmppkg + " (" + str(i+1) + "/" + str(len(pkglist)) + ")"
                                gliportage.copy_pkg_to_chroot(tmppkg)
                        gliportage.add_pkg_to_world(pkg)
                print "GRP install complete!"
        else:
                print "You must specify an operating mode (-g or -s)!"
                usage(progname)
                sys.exit(1)
1	"""
2	# Copyright 1999-2005 Gentoo Foundation
3	# This source code is distributed under the terms of version 2 of the GNU
4	# General Public License as published by the Free Software Foundation, a copy
5	# of which can be found in the main directory of this project.
6	Gentoo Linux Installer
7
8	$Id: GLIPortage.py,v 1.55 2006/09/13 16:50:23 agaffney Exp $
9	"""
10
11	import re
12	import os
13	import sys
14	import GLIUtility
15	from GLIException import GLIException
16
17	class GLIPortage(object):
18
19	def __init__(self, chroot_dir, grp_install, logger, debug, cc, compile_logfile):
20	self._chroot_dir = chroot_dir
21	self._grp_install = grp_install
22	self._logger = logger
23	self._debug = debug
24	self._cc = cc
25	self._compile_logfile = compile_logfile
26
27	def get_deps(self, pkgs):
28	pkglist = []
29	if isinstance(pkgs, str):
30	pkgs = pkgs.split()
31	for pkg in pkgs:
32	if not pkg: continue
33	if self._debug: self._logger.log("get_deps(): pkg is " + pkg)
34	if not self._grp_install or not self.get_best_version_vdb(pkg):
35	if self._debug: self._logger.log("get_deps(): grabbing compile deps")
36	tmppkglist = GLIUtility.spawn("emerge -p " + pkg + r" 2>/dev/null \| grep -e '^\[[a-z]' \| cut -d ']' -f2 \| sed -e 's:^ ::' -e 's: .\+$::'", chroot=self._chroot_dir, return_output=True)[1].strip().split("\n")
37	else:
38	if self._debug: self._logger.log("get_deps(): grabbing binary deps")
39	# The runtimedeps.py script generates a package install order that is very different from emerge itself
40	# tmppkglist = GLIUtility.spawn("python ../../runtimedeps.py " + self._chroot_dir + " " + pkg, return_output=True)[1].strip().split("\n")
41	tmppkglist = []
42	for tmppkg in GLIUtility.spawn("emerge -p " + pkg + r" 2>/dev/null \| grep -e '^\[[a-z]' \| cut -d ']' -f2 \| sed -e 's:^ ::' -e 's: .\+$::'", chroot=self._chroot_dir, return_output=True)[1].strip().split("\n"):
43	if self._debug: self._logger.log("get_deps(): looking at " + tmppkg)
44	if self.get_best_version_vdb("=" + tmppkg):
45	if self._debug: self._logger.log("get_deps(): package " + tmppkg + " in host vdb...adding to tmppkglist")
46	tmppkglist.append(tmppkg)
47	if self._debug: self._logger.log("get_deps(): deplist for " + pkg + ": " + str(tmppkglist))
48	for tmppkg in tmppkglist:
49	if self._debug: self._logger.log("get_deps(): checking to see if " + tmppkg + " is already in pkglist")
50	if not tmppkg in pkglist and not self.get_best_version_vdb_chroot("=" + tmppkg):
51	if self._debug: self._logger.log("get_deps(): adding " + tmppkg + " to pkglist")
52	pkglist.append(tmppkg)
53	if self._debug: self._logger.log("get_deps(): pkglist is " + str(pkglist))
54	if not pkglist:
55	raise GLIException("GetDepListError", 'fatal', 'get_deps', "Dep list is empty. This usually means there is no portage tree in the chroot")
56	return pkglist
57
58	def parse_vdb_contents(self, file):
59	entries = []
60	try:
61	vdbfile = open(file, "r")
62	except:
63	return entries
64	for line in vdbfile.readlines():
65	parts = line.strip().split(" ")
66	if parts[0] == "obj":
67	entries.append(parts[1])
68	# elif parts[0] == "dir":
69	# entries.append(parts[1] + "/")
70	elif parts[0] == "sym":
71	entries.append(" ".join(parts[1:4]))
72	entries.sort()
73	return entries
74
75	def copy_pkg_to_chroot(self, package, use_root=False, ignore_missing=False):
76	symlinks = { '/bin': '/mnt/livecd/bin/', '/boot': '/mnt/livecd/boot/', '/lib': '/mnt/livecd/lib/',
77	'/opt': '/mnt/livecd/opt/', '/sbin': '/mnt/livecd/sbin/', '/usr': '/mnt/livecd/usr/',
78	'/etc/gconf': '/usr/livecd/gconf/' }
79
80	tmpdir = "/var/tmp"
81	image_dir = tmpdir + "/portage/" + package.split("/")[1] + "/image"
82	root_cmd = ""
83	tmp_chroot_dir = self._chroot_dir
84	portage_tmpdir = "/var/tmp"
85	vdb_dir = "/var/db/pkg/"
86	if use_root:
87	root_cmd = "ROOT=" + self._chroot_dir
88	tmp_chroot_dir = ""
89	portage_tmpdir = self._chroot_dir + "/var/tmp"
90	vdb_dir = self._chroot_dir + "/var/db/pkg/"
91
92	# Create /tmp, /var/tmp, and /var/lib/portage with proper permissions
93	oldumask = os.umask(0)
94	if not os.path.exists(self._chroot_dir + "/tmp"):
95	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): /tmp doesn't exist in chroot...creating with proper permissions")
96	try:
97	os.mkdir(self._chroot_dir + "/tmp", 01777)
98	except:
99	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Failed to create /tmp in chroot")
100	if not os.path.exists(self._chroot_dir + "/var/tmp"):
101	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): /var/tmp doesn't exist in chroot...creating with proper permissions")
102	try:
103	os.mkdir(self._chroot_dir + "/var", 0755)
104	except:
105	pass
106	try:
107	os.mkdir(self._chroot_dir + "/var/tmp", 01777)
108	except:
109	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Failed to create /var/tmp in chroot")
110	if not os.path.exists(self._chroot_dir + "/var/lib/portage"):
111	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): /var/lib/portage doesn't exist in chroot...creating with proper permissions")
112	try:
113	os.mkdir(self._chroot_dir + "/var/lib", 0755)
114	except:
115	pass
116	try:
117	os.mkdir(self._chroot_dir + "/var/lib/portage", 02750)
118	except:
119	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Failed to create /var/lib/portage in chroot")
120	os.umask(oldumask)
121
122	# Check to see if package is actually in vdb
123	if not GLIUtility.is_file("/var/db/pkg/" + package):
124	if ignore_missing:
125	if self._debug:
126	self._logger.log("DEBUG: copy_pkg_to_chroot(): package " + package + " does not have a vdb entry but ignore_missing=True...ignoring error")
127	return
128	else:
129	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "There is no vdb entry for " + package)
130
131	# Copy the vdb entry for the package from the LiveCD to the chroot
132	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): copying vdb entry for " + package)
133	if not GLIUtility.exitsuccess(GLIUtility.spawn("mkdir -p " + self._chroot_dir + "/var/db/pkg/" + package + " && cp -a /var/db/pkg/" + package + "/* " + self._chroot_dir + "/var/db/pkg/" + package, logfile=self._compile_logfile, append_log=True)):
134	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not copy vdb entry for " + package)
135
136	# Create the image dir in the chroot
137	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running 'mkdir -p " + self._chroot_dir + image_dir + "'")
138	if not GLIUtility.exitsuccess(GLIUtility.spawn("mkdir -p " + self._chroot_dir + image_dir, logfile=self._compile_logfile, append_log=True)):
139	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not create image dir for " + package)
140
141	# Create list of files for tar to work with from CONTENTS file in vdb entry
142	entries = self.parse_vdb_contents("/var/db/pkg/" + package + "/CONTENTS")
143	if not entries:
144	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): no files for " + package + "...skipping tar and symlink fixup")
145	else:
146	# if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot: files for " + package + ": " + str(entries))
147	try:
148	tarfiles = open("/tmp/tarfilelist", "w")
149	for entry in entries:
150	parts = entry.split(" ")
151	# # Hack for symlink crappiness
152	# for symlink in symlinks:
153	# if parts[0].startswith(symlink):
154	# parts[0] = symlinks[symlink] + parts[0][len(symlink):]
155	tarfiles.write(parts[0] + "\n")
156	tarfiles.close()
157	except:
158	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not create filelist for " + package)
159
160	# Use tar to transfer files into IMAGE directory
161	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running 'tar -cp --files-from=/tmp/tarfilelist --no-recursion 2>/dev/null \| tar -C " + self._chroot_dir + image_dir + " -xp'")
162	if not GLIUtility.exitsuccess(GLIUtility.spawn("tar -cp --files-from=/tmp/tarfilelist --no-recursion 2>/dev/null \| tar -C " + self._chroot_dir + image_dir + " -xp", logfile=self._compile_logfile, append_log=True)):
163	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not execute tar for " + package)
164
165	# Fix mode, uid, and gid of directories
166	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running find " + self._chroot_dir + image_dir + " -mindepth 1 -type d 2>/dev/null \| sed -e 's:^" + self._chroot_dir + image_dir + "::' \| grep -v '^$'")
167	dirlist = GLIUtility.spawn("find " + self._chroot_dir + image_dir + " -mindepth 1 -type d 2>/dev/null \| sed -e 's:^" + self._chroot_dir + image_dir + "::' \| grep -v '^$'", return_output=True)[1].strip().split("\n")
168	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): found the following directories: " + str(dirlist))
169	if not dirlist or dirlist[0] == "":
170	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "directory list entry for " + package + "...this shouldn't happen!")
171	for dir in dirlist:
172	dirstat = os.stat(dir)
173	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): setting mode " + str(dirstat[0]) + " and uid/gid " + str(dirstat[4]) + "/" + str(dirstat[5]) + " for directory " + self._chroot_dir + image_dir + dir)
174	os.chown(self._chroot_dir + image_dir + dir, dirstat[4], dirstat[5])
175	os.chmod(self._chroot_dir + image_dir + dir, dirstat[0])
176
177	# # More symlink crappiness hacks
178	# for symlink in symlinks:
179	## if GLIUtility.is_file(self._chroot_dir + image_dir + symlinks[symlink]):
180	# if os.path.islink(self._chroot_dir + image_dir + symlink):
181	# if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): fixing " + symlink + " symlink ickiness stuff in " + image_dir + " for " + package)
182	# GLIUtility.spawn("rm " + self._chroot_dir + image_dir + symlink)
183	# if not GLIUtility.exitsuccess(GLIUtility.spawn("mv " + self._chroot_dir + image_dir + symlinks[symlink] + " " + self._chroot_dir + image_dir + symlink)):
184	# raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not fix " + symlink + " symlink ickiness for " + package)
185
186	# Run pkg_setup
187	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running pkg_setup for " + package)
188	if not GLIUtility.exitsuccess(GLIUtility.spawn("env " + root_cmd + " PORTAGE_TMPDIR=" + portage_tmpdir + " FEATURES=noauto ebuild " + vdb_dir + package + "/*.ebuild setup", chroot=tmp_chroot_dir, logfile=self._compile_logfile, append_log=True)):
189	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not execute pkg_setup for " + package)
190
191	# Run pkg_preinst
192	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running preinst for " + package)
193	if not GLIUtility.exitsuccess(GLIUtility.spawn("env " + root_cmd + " PORTAGE_TMPDIR=" + portage_tmpdir + " FEATURES=noauto ebuild " + vdb_dir + package + "/*.ebuild preinst", chroot=tmp_chroot_dir, logfile=self._compile_logfile, append_log=True)):
194	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not execute preinst for " + package)
195
196	# Copy files from image_dir to chroot
197	if not entries:
198	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): no files for " + package + "...skipping copy from image dir to /")
199	else:
200	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): copying files from " + image_dir + " to / for " + package)
201	# if not GLIUtility.exitsuccess(GLIUtility.spawn("cp -a " + self._chroot_dir + image_dir + "/* " + self._chroot_dir)):
202	if not GLIUtility.exitsuccess(GLIUtility.spawn("tar -C " + self._chroot_dir + image_dir + "/ -cp . \| tar -C " + self._chroot_dir + "/ -xp", logfile=self._compile_logfile, append_log=True)):
203	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not copy files from " + image_dir + " to / for " + package)
204
205	# Run pkg_postinst
206	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running postinst for " + package)
207	if not GLIUtility.exitsuccess(GLIUtility.spawn("env " + root_cmd + " PORTAGE_TMPDIR=" + portage_tmpdir + " FEATURES=noauto ebuild " + vdb_dir + package + "/*.ebuild postinst", chroot=tmp_chroot_dir, logfile=self._compile_logfile, append_log=True)):
208	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not execute postinst for " + package)
209
210	# Remove image_dir
211	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): removing " + image_dir + " for " + package)
212	if not GLIUtility.exitsuccess(GLIUtility.spawn("rm -rf " + self._chroot_dir + image_dir, logfile=self._compile_logfile, append_log=True)):
213	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not remove + " + image_dir + " for " + package)
214
215	# Run env-update
216	if not use_root:
217	if self._debug: self._logger.log("DEBUG: copy_pkg_to_chroot(): running env-update inside chroot")
218	if not GLIUtility.exitsuccess(GLIUtility.spawn("env-update", chroot=self._chroot_dir, logfile=self._compile_logfile, append_log=True)):
219	raise GLIException("CopyPackageToChrootError", 'fatal', 'copy_pkg_to_chroot', "Could not run env-update for " + package)
220
221	def add_pkg_to_world(self, package):
222	if package.find("/") == -1:
223	package = self.get_best_version_vdb_chroot(package)
224	if not package: return False
225	expr = re.compile('^=?(.+?/.+?)(-\d.+)?$')
226	res = expr.match(package)
227	if res:
228	GLIUtility.spawn("echo " + res.group(1) + " >> " + self._chroot_dir + "/var/lib/portage/world")
229
230	def get_best_version_vdb(self, package):
231	if package.startswith('='):
232	package = package[1:]
233	if GLIUtility.is_file("/var/db/pkg/" + package):
234	return package
235	else:
236	return ""
237	else:
238	return GLIUtility.spawn("portageq best_version / " + package, return_output=True)[1].strip()
239
240	def get_best_version_vdb_chroot(self, package):
241	if package.startswith('='):
242	package = package[1:]
243	if GLIUtility.is_file(self._chroot_dir + "/var/db/pkg/" + package):
244	return package
245	else:
246	return ""
247	else:
248	return GLIUtility.spawn("portageq best_version / " + package, chroot=self._chroot_dir, return_output=True)[1].strip()
249
250	# def get_best_version_tree(self, package):
251	# return portage.best(tree.match(package))
252
253	def emerge(self, packages, add_to_world=True):
254	if isinstance(packages, str):
255	packages = packages.split()
256	self._cc.addNotification("progress", (0, "Calculating dependencies for " + " ".join(packages)))
257	pkglist = self.get_deps(packages)
258	if self._debug: self._logger.log("install_packages(): pkglist is " + str(pkglist))
259	for i, pkg in enumerate(pkglist):
260	if not pkg: continue
261	if self._debug: self._logger.log("install_packages(): processing package " + pkg)
262	self._cc.addNotification("progress", (float(i) / len(pkglist), "Emerging " + pkg + " (" + str(i+1) + "/" + str(len(pkglist)) + ")"))
263	if not self._grp_install or not self.get_best_version_vdb("=" + pkg):
264	status = GLIUtility.spawn("emerge -1 =" + pkg, display_on_tty8=True, chroot=self._chroot_dir, logfile=self._compile_logfile, append_log=True)
265	# status = self._emerge("=" + pkg)
266	if not GLIUtility.exitsuccess(status):
267	raise GLIException("EmergePackageError", "fatal", "emerge", "Could not emerge " + pkg + "!")
268	else:
269	# try:
270	self.copy_pkg_to_chroot(pkg)
271	# except:
272	# raise GLIException("EmergePackageError", "fatal", "emerge", "Could not emerge " + pkg + "!")
273	self._cc.addNotification("progress", (float(i+1) / len(pkglist), "Done emerging " + pkg + " (" + str(i+1) + "/" + str(len(pkglist)) + ")"))
274	if add_to_world:
275	for package in packages:
276	self.add_pkg_to_world(package)
277
278
279	def usage(progname):
280	print """
281	Usage: %s [-c\|--chroot-dir <chroot directory>] [-g\|--grp] [-s\|--stage3] [-h\|--help]
282
283	Options:
284
285	-c\|--chroot-dir Specifies the directory where your chroot is. This is
286	"/mnt/gentoo" by default.
287
288	-g\|--grp Install specified packages and dependencies into chroot
289	by using files from the LiveCD.
290
291	-s\|--stage3 Create a stage3 equivelant in the chroot directory by using
292	files from the LiveCD.
293
294	-h\|--help Display this help
295	""" % (progname)
296
297	if __name__ == "__main__":
298	chroot_dir = "/mnt/gentoo"
299	mode = None
300	grp_packages = []
301	progname = sys.argv.pop(0)
302	while len(sys.argv):
303	arg = sys.argv.pop(0)
304	if arg == "-c" or arg == "--chroot-dir":
305	chroot_dir = sys.argv.pop(0)
306	elif arg == "-g" or arg == "--grp":
307	mode = "grp"
308	elif arg == "-s" or arg == "--stage3":
309	mode = "stage3"
310	elif arg == "-h" or arg == "--help":
311	usage(progname)
312	sys.exit(0)
313	elif arg[0] == "-":
314	usage(progname)
315	sys.exit(1)
316	else:
317	grp_packages.append(arg)
318
319	gliportage = GLIPortage(chroot_dir, True, None, False, None, None)
320	if mode == "stage3":
321	if not GLIUtility.is_file("/usr/livecd/systempkgs.txt"):
322	print "Required file /usr/livecd/systempkgs.txt does not exist!"
323	sys.exit(1)
324	try:
325	syspkgs = open("/usr/livecd/systempkgs.txt", "r")
326	systempkgs = syspkgs.readlines()
327	syspkgs.close()
328	except:
329	print "Could not open /usr/livecd/systempkgs.txt!"
330	sys.exit(1)
331
332	# Pre-create /lib (and possible /lib32 and /lib64)
333	if os.path.islink("/lib") and os.readlink("/lib") == "lib64":
334	if not GLIUtility.exitsuccess(GLIUtility.spawn("mkdir " + chroot_dir + "/lib64 && ln -s lib64 " + chroot_dir + "/lib")):
335	print "Could not precreate /lib64 dir and /lib -> /lib64 symlink"
336	sys.exit(1)
337
338	syspkglen = len(systempkgs)
339	for i, pkg in enumerate(systempkgs):
340	pkg = pkg.strip()
341	print "Copying " + pkg + " (" + str(i+1) + "/" + str(syspkglen) + ")"
342	gliportage.copy_pkg_to_chroot(pkg, True, ignore_missing=True)
343	GLIUtility.spawn("cp /etc/make.conf " + chroot_dir + "/etc/make.conf")
344	GLIUtility.spawn("ln -s `readlink /etc/make.profile` " + chroot_dir + "/etc/make.profile")
345	GLIUtility.spawn("cp -f /etc/inittab.old " + chroot_dir + "/etc/inittab")
346
347	# Nasty, nasty, nasty hack because vapier is a tool
348	for tmpfile in ("/etc/passwd", "/etc/group", "/etc/shadow"):
349	GLIUtility.spawn("grep -ve '^gentoo' " + tmpfile + " > " + chroot_dir + tmpfile)
350
351	chrootscript = r"""
352	#!/bin/bash
353
354	source /etc/make.conf
355	export LDPATH="/usr/lib/gcc-lib/${CHOST}/$(cd /usr/lib/gcc-lib/${CHOST} && ls -1 \| head -n 1)"
356
357	ldconfig $LDPATH
358	gcc-config 1
359	env-update
360	source /etc/profile
361	modules-update
362	[ -f /usr/bin/binutils-config ] && binutils-config 1
363	source /etc/profile
364	#mount -t proc none /proc
365	#cd /dev
366	#/sbin/MAKEDEV generic-i386
367	#umount /proc
368	[ -f /lib/udev-state/devices.tar.bz2 ] && tar -C /dev -xjf /lib/udev-state/devices.tar.bz2
369	"""
370	script = open(chroot_dir + "/tmp/extrastuff.sh", "w")
371	script.write(chrootscript)
372	script.close()
373	GLIUtility.spawn("chmod 755 /tmp/extrastuff.sh && /tmp/extrastuff.sh", chroot=chroot_dir)
374	GLIUtility.spawn("rm -rf /var/tmp/portage/* /usr/portage /tmp/*", chroot=chroot_dir)
375	print "Stage3 equivelant generation complete!"
376	elif mode == "grp":
377	for pkg in grp_packages:
378	if not gliportage.get_best_version_vdb(pkg):
379	print "Package " + pkg + " is not available for install from the LiveCD"
380	continue
381	pkglist = gliportage.get_deps(pkg)
382	for i, tmppkg in enumerate(pkglist):
383	print "Copying " + tmppkg + " (" + str(i+1) + "/" + str(len(pkglist)) + ")"
384	gliportage.copy_pkg_to_chroot(tmppkg)
385	gliportage.add_pkg_to_world(pkg)
386	print "GRP install complete!"
387	else:
388	print "You must specify an operating mode (-g or -s)!"
389	usage(progname)
390	sys.exit(1)