path: root/scripts/oe-pkgdata-util

#!/usr/bin/env python

# OpenEmbedded pkgdata utility
#
# Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Copyright 2012-2013 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#

import sys
import os
import os.path
import fnmatch
import re
import optparse
from collections import defaultdict

def glob(args, usage, debug=False):
    if len(args) < 3:
        usage()
        sys.exit(1)

    pkgdata_dir = args[0]
    pkglist_file = args[1]
    globs = args[2].split()

    if not os.path.exists(pkgdata_dir):
        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
        sys.exit(1)

    if not os.path.exists(pkglist_file):
        print('ERROR: Unable to find package list file %s' % pkglist_file)
        sys.exit(1)

    skipregex = re.compile("-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-")

    mappedpkgs = set()
    with open(pkglist_file, 'r') as f:
        for line in f:
            fields = line.rstrip().split()
            if not fields:
                continue
            pkg = fields[0]
            # We don't care about other args (used to need the package architecture but the
            # new pkgdata structure avoids the need for that)

            # Skip packages for which there is no point applying globs
            if skipregex.search(pkg):
                if debug:
                    print("%s -> !!" % pkg)
                continue

            # Skip packages that already match the globs, so if e.g. a dev package
            # is already installed and thus in the list, we don't process it any further
            # Most of these will be caught by skipregex already, but just in case...
            already = False
            for g in globs:
                if fnmatch.fnmatchcase(pkg, g):
                    already = True
                    break
            if already:
                if debug:
                    print("%s -> !" % pkg)
                continue

            # Define some functions
            def revpkgdata(pkgn):
                return os.path.join(pkgdata_dir, "runtime-reverse", pkgn)
            def fwdpkgdata(pkgn):
                return os.path.join(pkgdata_dir, "runtime", pkgn)
            def readpn(pkgdata_file):
                pn = ""
                with open(pkgdata_file, 'r') as f:
                    for line in f:
                        if line.startswith("PN:"):
                            pn = line.split(': ')[1].rstrip()
                return pn
            def readrenamed(pkgdata_file):
                renamed = ""
                pn = os.path.basename(pkgdata_file)
                with open(pkgdata_file, 'r') as f:
                    for line in f:
                        if line.startswith("PKG_%s:" % pn):
                            renamed = line.split(': ')[1].rstrip()
                return renamed

            # Main processing loop
            for g in globs:
                mappedpkg = ""
                # First just try substitution (i.e. packagename -> packagename-dev)
                newpkg = g.replace("*", pkg)
                revlink = revpkgdata(newpkg)
                if os.path.exists(revlink):
                    mappedpkg = os.path.basename(os.readlink(revlink))
                    fwdfile = fwdpkgdata(mappedpkg)
                    if os.path.exists(fwdfile):
                        mappedpkg = readrenamed(fwdfile)
                    if not os.path.exists(fwdfile + ".packaged"):
                        mappedpkg = ""
                else:
                    revlink = revpkgdata(pkg)
                    if os.path.exists(revlink):
                        # Check if we can map after undoing the package renaming (by resolving the symlink)
                        origpkg = os.path.basename(os.readlink(revlink))
                        newpkg = g.replace("*", origpkg)
                        fwdfile = fwdpkgdata(newpkg)
                        if os.path.exists(fwdfile):
                            mappedpkg = readrenamed(fwdfile)
                        else:
                            # That didn't work, so now get the PN, substitute that, then map in the other direction
                            pn = readpn(revlink)
                            newpkg = g.replace("*", pn)
                            fwdfile = fwdpkgdata(newpkg)
                            if os.path.exists(fwdfile):
                                mappedpkg = readrenamed(fwdfile)
                        if not os.path.exists(fwdfile + ".packaged"):
                            mappedpkg = ""
                    else:
                        # Package doesn't even exist...
                        if debug:
                            print "%s is not a valid package!" % (pkg)
                        break

                if mappedpkg:
                    if debug:
                        print "%s (%s) -> %s" % (pkg, g, mappedpkg)
                    mappedpkgs.add(mappedpkg)
                else:
                    if debug:
                        print "%s (%s) -> ?" % (pkg, g)

    if debug:
        print "------"

    print("\n".join(mappedpkgs))

def read_value(args, usage, debug=False):
    if len(args) < 3:
        usage()
        sys.exit(1)

    pkgdata_dir = args[0]
    var = args[1]
    packages = args[2].split()

    if not os.path.exists(pkgdata_dir):
        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
        sys.exit(1)

    def readvar(pkgdata_file, var):
        val = ""
        with open(pkgdata_file, 'r') as f:
            for line in f:
                if line.startswith(var + ":"):
                    val = line.split(': ')[1].rstrip()
        return val

    if debug:
        print "read-value('%s', '%s' '%s'" % (pkgdata_dir, var, packages)
    for package in packages:
        pkg_split = package.split('_')
        pkg_name = pkg_split[0]
        if debug:
            print "package: '%s'" % pkg_name
        revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg_name)
        if debug:
            print(revlink)
        if os.path.exists(revlink):
            mappedpkg = os.path.basename(os.readlink(revlink))
            qvar = var
            if qvar == "PKGSIZE":
                # append packagename
                qvar = "%s_%s" % (var, mappedpkg)
                # PKGSIZE is now in bytes, but we we want it in KB
                pkgsize = (int(readvar(revlink, qvar)) + 1024 // 2) // 1024
                print("%d" % pkgsize)
            else:
                print(readvar(revlink, qvar))

def lookup_pkg(args, usage, debug=False):
    if len(args) < 2:
        usage()
        sys.exit(1)

    pkgdata_dir = args[0]
    pkgs = args[1].split()

    if not os.path.exists(pkgdata_dir):
        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
        sys.exit(1)

    mappings = defaultdict(list)
    for pkg in pkgs:
        pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg)
        if os.path.exists(pkgfile):
            with open(pkgfile, 'r') as f:
                for line in f:
                    fields = line.rstrip().split(': ')
                    if fields[0] == 'PKG_%s' % pkg:
                        mappings[pkg].append(fields[1])
                        break
    if len(mappings) < len(pkgs):
        missing = list(set(pkgs) - set(mappings.keys()))
        sys.stderr.write("ERROR: the following packages could not be found: %s\n" % ', '.join(missing))
        sys.exit(1)

    items = []
    for pkg in pkgs:
        items.extend(mappings.get(pkg, []))
    print '\n'.join(items)

def lookup_recipe(args, usage, debug=False):
    if len(args) < 2:
        usage()
        sys.exit(1)

    pkgdata_dir = args[0]
    pkgs = args[1].split()

    if not os.path.exists(pkgdata_dir):
        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
        sys.exit(1)

    mappings = defaultdict(list)
    for pkg in pkgs:
        pkgfile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
        if os.path.exists(pkgfile):
            with open(pkgfile, 'r') as f:
                for line in f:
                    fields = line.rstrip().split(': ')
                    if fields[0] == 'PN':
                        mappings[pkg].append(fields[1])
                        break
    if len(mappings) < len(pkgs):
        missing = list(set(pkgs) - set(mappings.keys()))
        sys.stderr.write("ERROR: the following packages could not be found: %s\n" % ', '.join(missing))
        sys.exit(1)

    items = []
    for pkg in pkgs:
        items.extend(mappings.get(pkg, []))
    print '\n'.join(items)

def find_path(args, usage, debug=False):
    if len(args) < 2:
        usage()
        sys.exit(1)

    pkgdata_dir = args[0]
    targetpath = args[1]

    if not os.path.exists(pkgdata_dir):
        print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
        sys.exit(1)

    import json
    import fnmatch

    for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
        for fn in files:
            with open(os.path.join(root,fn)) as f:
                for line in f:
                    if line.startswith('FILES_INFO:'):
                        val = line.split(':', 1)[1].strip()
                        dictval = json.loads(val)
                        for fullpth in dictval.keys():
                            if fnmatch.fnmatchcase(fullpth, targetpath):
                                print("%s: %s" % (fn, fullpth))
                        break


def main():
    parser = optparse.OptionParser(
        usage = '''%prog [options] <command> <arguments>

Available commands:
    glob <pkgdatadir> <pkglistfile> "<globs>"
        expand one or more glob expressions over the packages listed in
        pkglistfile (one package per line)
    lookup-pkg <pkgdatadir> "<recipe-pkgs>"
        look up the specified recipe-space package name(s) to see what the
        final runtime package name is (e.g. eglibc becomes libc6)
    lookup-recipe <pkgdatadir> "<pkgs>"
        look up the specified package(s) to see which recipe they were
        produced by
    find-path <pkgdatadir> <path>
        find the package providing the specified path (wildcards * ? allowed)
    read-value <pkgdatadir> <value-name> "<pkgs>"
        read the named value from the pkgdata files for the specified
        packages''')

    parser.add_option("-d", "--debug",
            help = "Enable debug output",
            action="store_true", dest="debug", default=False)

    options, args = parser.parse_args(sys.argv)
    args = args[1:]

    if len(args) < 1:
        parser.print_help()
        sys.exit(1)

    if args[0] == "glob":
        glob(args[1:], parser.print_help, options.debug)
    elif args[0] == "lookup-pkg":
        lookup_pkg(args[1:], parser.print_help, options.debug)
    elif args[0] == "lookup-recipe":
        lookup_recipe(args[1:], parser.print_help, options.debug)
    elif args[0] == "find-path":
        find_path(args[1:], parser.print_help, options.debug)
    elif args[0] == "read-value":
        read_value(args[1:], parser.print_help, options.debug)
    else:
        parser.print_help()
        sys.exit(1)


if __name__ == "__main__":
    main()