Rework installation of dev, dbg, doc, and locale packages

Use a similar mechanism that was previously used to install locales at
rootfs generation time to install other "complementary" packages (e.g.
*-dev packages) - i.e. install all of the explicitly requested packages
and their dependencies, then get a list of the packages that were
installed, and use that list to install the complementary packages. This
has been implemented by using a list of globs which should make it
easier to extend in future.

The previous locale package installation code assumed that the locale
packages did not have any dependencies that were not already installed;
now that we are installing non-locale packages this is no longer
correct. In practice only the rpm backend actually made use of this
assumption, so it needed to be changed to call into the existing package
backend code to do the complementary package installation rather than
calling rpm directly.

This fixes the doc-pkgs IMAGE_FEATURES feature to work correctly, and
also ensures that all dev/dbg packages get installed for
dev-pkgs/dbg-pkgs respectively even if the dependency chains between
those packages was not ensuring that already.

The code has also been adapted to work correctly with the new
SDK-from-image functionality. To that end, an SDKIMAGE_FEATURES variable
has been added to allow specifying what extra image features should go
into the SDK (extra, because by virtue of installing all of the packages
in the image into the target part of the SDK, we already include all of
IMAGE_FEATURES) with a default value of "dev-pkgs dbg-pkgs".

Fixes [YOCTO #2614].

(From OE-Core rev: 72d1048a8381fa4a8c4c0d082047536727b4be47)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

10 files changed, 390 insertions, 111 deletions

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 0a380f1172..0f7744aa5e 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -6,7 +6,8 @@ inherit imagetest-${IMAGETEST}
 inherit populate_sdk_base
 
 TOOLCHAIN_TARGET_TASK += "${PACKAGE_INSTALL}"
-TOOLCHAIN_TARGET_TASK_ATTEMPTONLY += "${PACKAGE_INSTALL_ATTEMPTONLY} ${PACKAGE_GROUP_dev-pkgs} ${PACKAGE_GROUP_dbg-pkgs}"
+TOOLCHAIN_TARGET_TASK_ATTEMPTONLY += "${PACKAGE_INSTALL_ATTEMPTONLY}"
+POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_install_complementary populate_sdk; "
 
 inherit gzipnative
 
@@ -38,25 +39,23 @@ def normal_groups(d):
     features = set(oe.data.typed_value('IMAGE_FEATURES', d))
     return features.difference(extras)
 
-def normal_pkgs_to_install(d):
-    import oe.packagedata
-
-    to_install = oe.data.typed_value('IMAGE_INSTALL', d)
-    features = normal_groups(d)
-    required = list(oe.packagegroup.required_packages(features, d))
-    optional = list(oe.packagegroup.optional_packages(features, d))
-    all_packages = to_install + required + optional
-
-    recipes = filter(None, [oe.packagedata.recipename(pkg, d) for pkg in all_packages])
-
-    return all_packages + recipes
-
-PACKAGE_GROUP_dbg-pkgs = "${@' '.join('%s-dbg' % pkg for pkg in normal_pkgs_to_install(d))}"
-PACKAGE_GROUP_dbg-pkgs[optional] = "1"
-PACKAGE_GROUP_dev-pkgs = "${@' '.join('%s-dev' % pkg for pkg in normal_pkgs_to_install(d))}"
-PACKAGE_GROUP_dev-pkgs[optional] = "1"
-PACKAGE_GROUP_doc-pkgs = "${@' '.join('%s-doc' % pkg for pkg in normal_pkgs_to_install(d))}"
-PACKAGE_GROUP_doc-pkgs[optional] = "1"
+# Wildcards specifying complementary packages to install for every package that has been explicitly
+# installed into the rootfs
+def complementary_globs(featurevar, d):
+    globs = []
+    features = set((d.getVar(featurevar, True) or '').split())
+    for feature in features:
+        if feature == 'dev-pkgs':
+            globs.append('*-dev')
+        elif feature == 'doc-pkgs':
+            globs.append('*-doc')
+        elif feature == 'dbg-pkgs':
+            globs.append('*-dbg')
+    return ' '.join(globs)
+
+IMAGE_INSTALL_COMPLEMENTARY = '${@complementary_globs("IMAGE_FEATURES", d)}'
+SDKIMAGE_FEATURES ??= "dev-pkgs dbg-pkgs"
+SDKIMAGE_INSTALL_COMPLEMENTARY = '${@complementary_globs("SDKIMAGE_FEATURES", d)}'
 
 # "export IMAGE_BASENAME" not supported at this time
 IMAGE_INSTALL ?= ""
@@ -306,32 +305,44 @@ get_split_linguas() {
     done | sort | uniq
 }
 
-rootfs_install_all_locales() {
-    # Generate list of installed packages for which additional locale packages might be available
-    INSTALLED_PACKAGES=`list_installed_packages | egrep -v -- "(-locale-|^locale-base-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)"`
-
-    # Generate a list of locale packages that exist
-    SPLIT_LINGUAS=`get_split_linguas`
-    PACKAGES_TO_INSTALL=""
-    for lang in $SPLIT_LINGUAS; do
-        for pkg in $INSTALLED_PACKAGES; do
-            existing_pkg=`rootfs_check_package_exists $pkg-locale-$lang`
-            if [ "$existing_pkg" != "" ]; then
-                PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $existing_pkg"
-            fi
+rootfs_install_complementary() {
+    # Install complementary packages based upon the list of currently installed packages
+    # e.g. locales, *-dev, *-dbg, etc. This will only attempt to install these packages,
+    # if they don't exist then no error will occur.
+    # Note: every backend needs to call this function explicitly after the normal
+    # package installation
+
+    # Get list of installed packages
+    list_installed_packages arch > ${WORKDIR}/installed_pkgs.txt
+
+    # Apply the globs to all the packages currently installed
+    if [ "$1" = "populate_sdk" ] ; then
+        GLOBS="${SDKIMAGE_INSTALL_COMPLEMENTARY}"
+    else
+        GLOBS="${IMAGE_INSTALL_COMPLEMENTARY}"
+        # Add locales
+        SPLIT_LINGUAS=`get_split_linguas`
+        PACKAGES_TO_INSTALL=""
+        for lang in $SPLIT_LINGUAS ; do
+            GLOBS="$GLOBS *-locale-$lang"
         done
-    done
+    fi
+
+    if [ "$GLOBS" != "" ] ; then
+        # Use the magic script to do all the work for us :)
+        oe-pkgdata-util glob ${TMPDIR}/pkgdata ${TARGET_VENDOR}-${TARGET_OS} ${WORKDIR}/installed_pkgs.txt "$GLOBS" > ${WORKDIR}/complementary_pkgs.txt
 
-    # Install the packages, if any
-    if [ "$PACKAGES_TO_INSTALL" != "" ]; then
-        rootfs_install_packages $PACKAGES_TO_INSTALL
+        # Install the packages, if any
+        sed -i '/^$/d' ${WORKDIR}/complementary_pkgs.txt
+        if [ -s ${WORKDIR}/complementary_pkgs.txt ]; then
+            echo "Installing complementary packages"
+            rootfs_install_packages ${WORKDIR}/complementary_pkgs.txt
+        fi
     fi
 
     # Workaround for broken shell function dependencies
     if false ; then
         get_split_linguas
-        list_installed_packages
-        rootfs_check_package_exists
     fi
 }
 
diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
index 29018e9cca..b4bc52e69b 100644
--- a/meta/classes/package_rpm.bbclass
+++ b/meta/classes/package_rpm.bbclass
@@ -248,6 +248,7 @@ process_pkg_list_rpm() {
 # INSTALL_PACKAGES_LINGUAS_RPM - additional packages for uclibc
 # INSTALL_PROVIDENAME_RPM - content for provide name
 # INSTALL_TASK_RPM - task name
+# INSTALL_COMPLEMENTARY_RPM - 1 to enable complementary package install mode
 
 package_install_internal_rpm () {
 
@@ -261,31 +262,35 @@ package_install_internal_rpm () {
 	local providename="${INSTALL_PROVIDENAME_RPM}"
 	local task="${INSTALL_TASK_RPM}"
 
-	# Setup base system configuration
-	mkdir -p ${target_rootfs}/etc/rpm/
-	echo "${platform}${TARGET_VENDOR}-${TARGET_OS}" > ${target_rootfs}/etc/rpm/platform
-	if [ ! -z "$platform_extra" ]; then
-		for pt in $platform_extra ; do
-			case $pt in
-				noarch | any | all)
-					os="`echo ${TARGET_OS} | sed "s,-.*,,"`.*"
-					;;
-				*)
-					os="${TARGET_OS}"
-					;;
-			esac
-			echo "$pt-.*-$os" >> ${target_rootfs}/etc/rpm/platform
-		done
-	fi
+	if [ "${INSTALL_COMPLEMENTARY_RPM}" != "1" ] ; then
+		# Setup base system configuration
+		mkdir -p ${target_rootfs}/etc/rpm/
+		echo "${platform}${TARGET_VENDOR}-${TARGET_OS}" > ${target_rootfs}/etc/rpm/platform
+		if [ ! -z "$platform_extra" ]; then
+			for pt in $platform_extra ; do
+				case $pt in
+					noarch | any | all)
+						os="`echo ${TARGET_OS} | sed "s,-.*,,"`.*"
+						;;
+					*)
+						os="${TARGET_OS}"
+						;;
+				esac
+				echo "$pt-.*-$os" >> ${target_rootfs}/etc/rpm/platform
+			done
+		fi
 
-	# Tell RPM that the "/" directory exist and is available
-	mkdir -p ${target_rootfs}/etc/rpm/sysinfo
-	echo "/" >${target_rootfs}/etc/rpm/sysinfo/Dirnames
-	if [ ! -z "$providename" ]; then
-		cat /dev/null > ${target_rootfs}/etc/rpm/sysinfo/Providename
-		for provide in $providename ; do
-			echo $provide >> ${target_rootfs}/etc/rpm/sysinfo/Providename
-		done
+		# Tell RPM that the "/" directory exist and is available
+		mkdir -p ${target_rootfs}/etc/rpm/sysinfo
+		echo "/" >${target_rootfs}/etc/rpm/sysinfo/Dirnames
+		if [ ! -z "$providename" ]; then
+			cat /dev/null > ${target_rootfs}/etc/rpm/sysinfo/Providename
+			for provide in $providename ; do
+				echo $provide >> ${target_rootfs}/etc/rpm/sysinfo/Providename
+			done
+		fi
+	else
+		mv ${target_rootfs}/install/total_solution.manifest ${target_rootfs}/install/original_solution.manifest
 	fi
 
 	# Setup manifest of packages to install...
@@ -480,13 +485,22 @@ mutex_set_max 163840
 # ================ Replication
 EOF
 
-	# RPM is special. It can't handle dependencies and preinstall scripts correctly. Its
-	# probably a feature. The only way to convince rpm to actually run the preinstall scripts 
-	# for base-passwd and shadow first before installing packages that depend on these packages 
-	# is to do two image installs, installing one set of packages, then the other.
-	if [ "${INC_RPM_IMAGE_GEN}" = "1" -a -f "$pre_btmanifest" ]; then
-		echo "Skipping pre install due to exisitng image"
+	if [ "${INSTALL_COMPLEMENTARY_RPM}" = "1" ] ; then
+		# Only install packages not already installed (dependency calculation will
+		# almost certainly have added some that have been)
+		sort ${target_rootfs}/install/original_solution.manifest > ${target_rootfs}/install/original_solution_sorted.manifest
+		sort ${target_rootfs}/install/total_solution.manifest > ${target_rootfs}/install/total_solution_sorted.manifest
+		comm -2 -3 ${target_rootfs}/install/total_solution_sorted.manifest \
+			${target_rootfs}/install/original_solution_sorted.manifest | awk '{print $1}' > \
+			${target_rootfs}/install/diff.manifest
+		mv ${target_rootfs}/install/diff.manifest ${target_rootfs}/install/total_solution.manifest
+	elif [ "${INC_RPM_IMAGE_GEN}" = "1" -a -f "$pre_btmanifest" ]; then
+		echo "Skipping pre install due to existing image"
 	else
+		# RPM is special. It can't handle dependencies and preinstall scripts correctly. Its
+		# probably a feature. The only way to convince rpm to actually run the preinstall scripts
+		# for base-passwd and shadow first before installing packages that depend on these packages
+		# is to do two image installs, installing one set of packages, then the other.
 		rm -f ${target_rootfs}/install/initial_install.manifest
 		echo "Installing base dependencies first (base-passwd, base-files and shadow) since rpm is special"
 		grep /base-passwd-[0-9] ${target_rootfs}/install/total_solution.manifest >> ${target_rootfs}/install/initial_install.manifest || true
diff --git a/meta/classes/populate_sdk_deb.bbclass b/meta/classes/populate_sdk_deb.bbclass
index 9e9e1e1d8b..6f89dcfa06 100644
--- a/meta/classes/populate_sdk_deb.bbclass
+++ b/meta/classes/populate_sdk_deb.bbclass
@@ -36,6 +36,8 @@ populate_sdk_deb () {
 
 	package_install_internal_deb
 
+	${POPULATE_SDK_POST_TARGET_COMMAND}
+
 	populate_sdk_post_deb ${INSTALL_ROOTFS_DEB}
 
 	populate_sdk_log_check populate_sdk
diff --git a/meta/classes/populate_sdk_ipk.bbclass b/meta/classes/populate_sdk_ipk.bbclass
index 4321afb295..65a95e7a2b 100644
--- a/meta/classes/populate_sdk_ipk.bbclass
+++ b/meta/classes/populate_sdk_ipk.bbclass
@@ -29,6 +29,8 @@ populate_sdk_ipk() {
 
 	package_install_internal_ipk
 
+	${POPULATE_SDK_POST_TARGET_COMMAND}
+
 	#install host
 	export INSTALL_ROOTFS_IPK="${SDK_OUTPUT}"
 	export INSTALL_CONF_IPK="${IPKGCONF_SDK}"
diff --git a/meta/classes/populate_sdk_rpm.bbclass b/meta/classes/populate_sdk_rpm.bbclass
index 365a337a05..fac653bd6f 100644
--- a/meta/classes/populate_sdk_rpm.bbclass
+++ b/meta/classes/populate_sdk_rpm.bbclass
@@ -37,6 +37,7 @@ populate_sdk_rpm () {
 	export INSTALL_PACKAGES_LINGUAS_RPM=""
 	export INSTALL_PROVIDENAME_RPM="/bin/sh /bin/bash /usr/bin/env /usr/bin/perl pkgconfig pkgconfig(pkg-config)"
 	export INSTALL_TASK_RPM="populate_sdk-target"
+	export INSTALL_COMPLEMENTARY_RPM=""
 
 	# Setup base system configuration
 	mkdir -p ${INSTALL_ROOTFS_RPM}/etc/rpm/
@@ -74,6 +75,7 @@ EOF
 	export INSTALL_PLATFORM_EXTRA_RPM
 
 	package_install_internal_rpm
+	${POPULATE_SDK_POST_TARGET_COMMAND}
 	populate_sdk_post_rpm ${INSTALL_ROOTFS_RPM}
 
 	## install nativesdk ##
@@ -86,6 +88,7 @@ EOF
 	export INSTALL_PACKAGES_LINGUAS_RPM=""
 	export INSTALL_PROVIDENAME_RPM="/bin/sh /bin/bash /usr/bin/env /usr/bin/perl pkgconfig libGL.so()(64bit) libGL.so"
 	export INSTALL_TASK_RPM="populate_sdk_rpm-nativesdk"
+	export INSTALL_COMPLEMENTARY_RPM=""
 
 	# List must be prefered to least preferred order
 	INSTALL_PLATFORM_EXTRA_RPM=""
diff --git a/meta/classes/rootfs_deb.bbclass b/meta/classes/rootfs_deb.bbclass
index 67871a9087..a002b1ec02 100644
--- a/meta/classes/rootfs_deb.bbclass
+++ b/meta/classes/rootfs_deb.bbclass
@@ -10,7 +10,7 @@ do_rootfs[recrdeptask] += "do_package_write_deb"
 
 do_rootfs[lockfiles] += "${WORKDIR}/deb.lock"
 
-DEB_POSTPROCESS_COMMANDS = "rootfs_install_all_locales; "
+DEB_POSTPROCESS_COMMANDS = ""
 
 opkglibdir = "${localstatedir}/lib/opkg"
 
@@ -42,6 +42,8 @@ fakeroot rootfs_deb_do_rootfs () {
 	package_install_internal_deb
 	${DEB_POSTPROCESS_COMMANDS}
 
+	rootfs_install_complementary
+
 	export D=${IMAGE_ROOTFS}
 	export OFFLINE_ROOT=${IMAGE_ROOTFS}
 	export IPKG_OFFLINE_ROOT=${IMAGE_ROOTFS}
@@ -87,10 +89,16 @@ remove_packaging_data_files() {
 	rm -rf ${IMAGE_ROOTFS}/usr/dpkg/
 }
 
-DPKG_QUERY_COMMAND = "${STAGING_BINDIR_NATIVE}/dpkg --admindir=${IMAGE_ROOTFS}/var/lib/dpkg"
+# This will of course only work after rootfs_deb_do_rootfs has been called
+DPKG_QUERY_COMMAND = "${STAGING_BINDIR_NATIVE}/dpkg-query --admindir=$INSTALL_ROOTFS_DEB/var/lib/dpkg"
 
 list_installed_packages() {
-	${DPKG_QUERY_COMMAND} -l | grep ^ii | awk '{ print $2 }'
+	if [ "$1" = "arch" ] ; then
+		# Here we want the PACKAGE_ARCH not the deb architecture
+		${DPKG_QUERY_COMMAND} -W -f='${Package} ${PackageArch}\n'
+	else
+		${DPKG_QUERY_COMMAND} -W -f='${Package}\n'
+	fi
 }
 
 get_package_filename() {
@@ -110,16 +118,9 @@ list_package_recommends() {
 	${DPKG_QUERY_COMMAND} -s $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
 }
 
-rootfs_check_package_exists() {
-	if [ `apt-cache policy $1 | wc -l` -gt 4 ]; then
-		echo $1
-	fi
-}
-
 rootfs_install_packages() {
-	${STAGING_BINDIR_NATIVE}/apt-get install $@ --force-yes --allow-unauthenticated
+	${STAGING_BINDIR_NATIVE}/apt-get install `cat $1` --force-yes --allow-unauthenticated
 
-	for pkg in $@ ; do
-		deb_package_setflag installed $pkg
-	done
+	# Mark all packages installed
+	sed -i -e "s/Status: install ok unpacked/Status: install ok installed/;" $INSTALL_ROOTFS_DEB/var/lib/dpkg/status
 }
diff --git a/meta/classes/rootfs_ipk.bbclass b/meta/classes/rootfs_ipk.bbclass
index 9732385613..7df97a014b 100644
--- a/meta/classes/rootfs_ipk.bbclass
+++ b/meta/classes/rootfs_ipk.bbclass
@@ -15,10 +15,12 @@ do_rootfs[recrdeptask] += "do_package_write_ipk"
 do_rootfs[lockfiles] += "${WORKDIR}/ipk.lock"
 
 IPKG_ARGS = "-f ${IPKGCONF_TARGET} -o ${IMAGE_ROOTFS} --force-overwrite"
+# The _POST version also works when constructing the matching SDK
+IPKG_ARGS_POST = "-f ${IPKGCONF_TARGET} -o $INSTALL_ROOTFS_IPK --force-overwrite"
 
 OPKG_PREPROCESS_COMMANDS = "package_update_index_ipk; package_generate_ipkg_conf"
 
-OPKG_POSTPROCESS_COMMANDS = "ipk_insert_feed_uris; rootfs_install_all_locales; "
+OPKG_POSTPROCESS_COMMANDS = "ipk_insert_feed_uris; "
 
 opkglibdir = "${localstatedir}/lib/opkg"
 
@@ -74,6 +76,8 @@ fakeroot rootfs_ipk_do_rootfs () {
 	#mkdir -p ${IMAGE_ROOTFS}/etc/opkg/
 	#grep "^arch" ${IPKGCONF_TARGET} >${IMAGE_ROOTFS}/etc/opkg/arch.conf
 
+	rootfs_install_complementary
+
 	${OPKG_POSTPROCESS_COMMANDS}
 	${ROOTFS_POSTINSTALL_COMMAND}
 	
@@ -125,12 +129,16 @@ remove_packaging_data_files() {
 }
 
 list_installed_packages() {
-	grep ^Package: ${IMAGE_ROOTFS}${opkglibdir}/status | sed "s/^Package: //"
+	if [ "$1" = "arch" ] ; then
+		opkg-cl ${IPKG_ARGS_POST} status | opkg-query-helper.py -a
+	else
+		opkg-cl ${IPKG_ARGS_POST} list_installed | awk '{ print $1 }'
+	fi
 }
 
 get_package_filename() {
 	set +x
-	info=`opkg-cl ${IPKG_ARGS} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" || true`
+	info=`opkg-cl ${IPKG_ARGS_POST} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" || true`
 	name=`echo "${info}" | awk '/^Package/ {printf $2"_"}'`
 	name=$name`echo "${info}" | awk -F: '/^Version/ {printf $NF"_"}' | sed 's/^\s*//g'`
 	name=$name`echo "${info}" | awk '/^Archi/ {print $2".ipk"}'`
@@ -145,21 +153,15 @@ get_package_filename() {
 }
 
 list_package_depends() {
-	opkg-cl ${IPKG_ARGS} info $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
+	opkg-cl ${IPKG_ARGS_POST} info $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
 }
 
 list_package_recommends() {
-	opkg-cl ${IPKG_ARGS} info $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
-}
-
-rootfs_check_package_exists() {
-	if [ `opkg-cl ${IPKG_ARGS} info $1 | wc -l` -gt 2 ]; then
-		echo $1
-	fi
+	opkg-cl ${IPKG_ARGS_POST} info $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
 }
 
 rootfs_install_packages() {
-	opkg-cl ${IPKG_ARGS} install $PACKAGES_TO_INSTALL
+	opkg-cl ${IPKG_ARGS_POST} install `cat $1`
 }
 
 ipk_insert_feed_uris () {
diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass
index cd9c5ab778..1cc4a84495 100644
--- a/meta/classes/rootfs_rpm.bbclass
+++ b/meta/classes/rootfs_rpm.bbclass
@@ -22,7 +22,7 @@ do_rootfs[depends] += "opkg-native:do_populate_sysroot"
 do_rootfs[recrdeptask] += "do_package_write_rpm"
 
 RPM_PREPROCESS_COMMANDS = "package_update_index_rpm; package_generate_rpm_conf; "
-RPM_POSTPROCESS_COMMANDS = "rootfs_install_all_locales; "
+RPM_POSTPROCESS_COMMANDS = ""
 
 # 
 # Allow distributions to alter when [postponed] package install scripts are run
@@ -56,6 +56,7 @@ fakeroot rootfs_rpm_do_rootfs () {
 	export INSTALL_PACKAGES_LINGUAS_RPM="${LINGUAS_INSTALL}"
 	export INSTALL_PROVIDENAME_RPM=""
 	export INSTALL_TASK_RPM="rootfs_rpm_do_rootfs"
+	export INSTALL_COMPLEMENTARY_RPM=""
 
 	# Setup base system configuration
 	mkdir -p ${INSTALL_ROOTFS_RPM}/etc/rpm/
@@ -69,6 +70,8 @@ fakeroot rootfs_rpm_do_rootfs () {
 
 	package_install_internal_rpm
 
+	rootfs_install_complementary
+
 	export D=${IMAGE_ROOTFS}
 	export OFFLINE_ROOT=${IMAGE_ROOTFS}
 	export IPKG_OFFLINE_ROOT=${IMAGE_ROOTFS}
@@ -134,11 +137,15 @@ remove_packaging_data_files() {
 	rm -rf ${IMAGE_ROOTFS}${opkglibdir}
 }
 
-RPM_QUERY_CMD = '${RPM} --root ${IMAGE_ROOTFS} -D "_dbpath ${rpmlibdir}" \
+RPM_QUERY_CMD = '${RPM} --root $INSTALL_ROOTFS_RPM -D "_dbpath ${rpmlibdir}" \
 		-D "__dbi_txn create nofsync private"'
 
 list_installed_packages() {
-	${RPM_QUERY_CMD} -qa --qf "[%{NAME}\n]"
+	if [ "$1" = "arch" ] ; then
+		${RPM_QUERY_CMD} -qa --qf "[%{NAME} %{ARCH}\n]"
+	else
+		${RPM_QUERY_CMD} -qa --qf "[%{NAME}\n]"
+	fi
 }
 
 get_package_filename() {
@@ -172,21 +179,15 @@ list_package_recommends() {
 	${RPM_QUERY_CMD} -q --suggests $1
 }
 
-rootfs_check_package_exists() {
-	resolve_package_rpm ${RPMCONF_TARGET_BASE}-base_archs.conf $1
-}
-
 rootfs_install_packages() {
-    # The pkg to be installed here is not controlled by the
-    # package_install_internal_rpm, so it may have already been
-    # installed(e.g, installed in the first time when generate the
-    # rootfs), use '--replacepkgs' to always install them
-	for pkg in $@; do
-		${RPM} --root ${IMAGE_ROOTFS} -D "_dbpath ${rpmlibdir}" \
-			-D "__dbi_txn create nofsync private" \
-			--noscripts --notriggers --noparentdirs --nolinktos \
-			--replacepkgs -Uhv $pkg || true
-	done
+	# Note - we expect the variables not set here to already have been set
+	export INSTALL_PACKAGES_RPM=""
+	export INSTALL_PACKAGES_ATTEMPTONLY_RPM="`cat $1`"
+	export INSTALL_PROVIDENAME_RPM=""
+	export INSTALL_TASK_RPM="rootfs_install_packages"
+	export INSTALL_COMPLEMENTARY_RPM="1"
+
+	package_install_internal_rpm
 }
 
 python () {
diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
new file mode 100755
index 0000000000..2427f10d89
--- /dev/null
+++ b/scripts/oe-pkgdata-util
@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+
+# OpenEmbedded pkgdata utility
+#
+# Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Copyright 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#
+# Currently only has one function - mapping of packages to their dev/dbg/doc/locale etc. 
+# counterparts ("glob" command). Could be extended in future to perform other useful querying
+# functions on the pkgdata though.
+#
+
+import sys
+import os
+import os.path
+import fnmatch
+import re
+
+def usage():
+    print("syntax: pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"");
+
+
+
+def glob(args):
+    if len(args) < 4:
+        usage()
+        sys.exit(1)
+
+    pkgdata_dir = args[0]
+    target_suffix = args[1]
+    pkglist_file = args[2]
+    globs = args[3].split()
+
+    if target_suffix.startswith("-"):
+        target_suffix = target_suffix[1:]
+
+    skipregex = re.compile("-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-")
+
+    mappedpkgs = set()
+    with open(pkglist_file, 'r') as f:
+        for line in f:
+            fields = line.rstrip().split()
+            if len(fields) < 2:
+                continue
+            pkg = fields[0]
+            arch = fields[1]
+            multimach_target_sys = "%s-%s" % (arch, target_suffix)
+
+            # Skip packages for which there is no point applying globs
+            if skipregex.search(pkg):
+                if debug:
+                    print("%s -> !!" % pkg)
+                continue
+
+            # Skip packages that already match the globs, so if e.g. a dev package
+            # is already installed and thus in the list, we don't process it any further
+            # Most of these will be caught by skipregex already, but just in case...
+            already = False
+            for g in globs:
+                if fnmatch.fnmatchcase(pkg, g):
+                    already = True
+                    break
+            if already:
+                if debug:
+                    print("%s -> !" % pkg)
+                continue
+
+            # Define some functions
+            def revpkgdata(pkgn):
+                return os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkgn)
+            def fwdpkgdata(pkgn):
+                return os.path.join(pkgdata_dir, multimach_target_sys, "runtime", pkgn)
+            def readpn(pkgdata_file):
+                pn = ""
+                with open(pkgdata_file, 'r') as f:
+                    for line in f:
+                        if line.startswith("PN:"):
+                            pn = line.split(': ')[1].rstrip()
+                return pn
+            def readrenamed(pkgdata_file):
+                renamed = ""
+                pn = os.path.basename(pkgdata_file)
+                with open(pkgdata_file, 'r') as f:
+                    for line in f:
+                        if line.startswith("PKG_%s:" % pn):
+                            renamed = line.split(': ')[1].rstrip()
+                return renamed
+
+            # Main processing loop
+            for g in globs:
+                mappedpkg = ""
+                # First just try substitution (i.e. packagename -> packagename-dev)
+                newpkg = g.replace("*", pkg)
+                revlink = revpkgdata(newpkg)
+                if os.path.exists(revlink):
+                    mappedpkg = os.path.basename(os.readlink(revlink))
+                    fwdfile = fwdpkgdata(mappedpkg)
+                    if os.path.exists(fwdfile):
+                        mappedpkg = readrenamed(fwdfile)
+                else:
+                    # That didn't work, so now get the PN, substitute that, then map in the other direction
+                    revlink = revpkgdata(pkg)
+                    if os.path.exists(revlink):
+                        pn = readpn(revlink)
+                        newpkg = g.replace("*", pn)
+                        fwdfile = fwdpkgdata(newpkg)
+                        if os.path.exists(fwdfile):
+                            mappedpkg = readrenamed(fwdfile)
+                    else:
+                        # Package doesn't even exist...
+                        if debug:
+                            print "%s is not a valid package!" % (pkg)
+                        break
+
+                if mappedpkg:
+                    if debug:
+                        print "%s (%s) -> %s" % (pkg, g, mappedpkg)
+                    mappedpkgs.add(mappedpkg)
+                else:
+                    if debug:
+                        print "%s (%s) -> ?" % (pkg, g)
+
+    if debug:
+        print "------"
+
+    print("\n".join(mappedpkgs))
+
+
+
+# Too lazy to use getopt
+debug = False
+noopt = False
+args = []
+for arg in sys.argv[1:]:
+    if arg == "--":
+        noopt = True
+    else:
+        if not noopt:
+            if arg == "-d":
+                debug = True
+                continue
+        args.append(arg)
+
+if len(args) < 1:
+    usage()
+    sys.exit(1)
+
+if args[0] == "glob":
+    glob(args[1:])
+else:
+    usage()
+    sys.exit(1)
diff --git a/scripts/opkg-query-helper.py b/scripts/opkg-query-helper.py
new file mode 100755
index 0000000000..b52284b325
--- /dev/null
+++ b/scripts/opkg-query-helper.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+# OpenEmbedded opkg query helper utility
+#
+# Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Copyright 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#
+
+
+import sys
+import fileinput
+import re
+
+archmode = False
+filemode = False
+
+args = []
+for arg in sys.argv[1:]:
+    if arg == '-a':
+        archmode = True
+    elif arg == '-f':
+        filemode = True
+    else:
+        args.append(arg)
+
+# Regex for removing version specs after dependency items
+verregex = re.compile(' \([=<>]* [^ )]*\)')
+
+pkg = ""
+ver = ""
+for line in fileinput.input(args):
+    line = line.rstrip()
+    if ': ' in line:
+        if line.startswith("Package:"):
+            pkg = line.split(": ")[1]
+            ver = ""
+        else:
+            if archmode:
+                if line.startswith("Architecture:"):
+                    arch = line.split(": ")[1]
+                    print("%s %s" % (pkg,arch))
+            elif filemode:
+                if line.startswith("Version:"):
+                    ver = line.split(": ")[1]
+                elif line.startswith("Architecture:"):
+                    arch = line.split(": ")[1]
+                    print("%s %s_%s_%s.ipk" % (pkg,pkg,ver,arch))
+            else:
+                if line.startswith("Depends:"):
+                    depval = line.split(": ")[1]
+                    deps = depval.split(", ")
+                    for dep in deps:
+                        dep = verregex.sub('', dep)
+                        print("%s|%s" % (pkg,dep))
+                elif line.startswith("Recommends:"):
+                    recval = line.split(": ")[1]
+                    recs = recval.split(", ")
+                    for rec in recs:
+                        rec = verregex.sub('', rec)
+                        print("%s|%s [REC]" % (pkg, rec))
+