From 95d4da332940da7ff4d799f18ab0167d3631e01a Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Thu, 4 Sep 2014 15:27:32 +0100 Subject: bitbake: toaster: enable SSH-based remote build support We enable support for starting builds on remote machines through SSH. The support is limited to poky-based distributions. We refactor localhost build support and we update bldcontrol application tests to uniformely test the APIs of localhost and SSH build controllers. [YOCTO #6240] (Bitbake rev: c2ad9c9bb83f61c171434324df8c4d5ee655a556) Signed-off-by: Alexandru DAMIAN Signed-off-by: Richard Purdie --- bitbake/lib/toaster/bldcontrol/bbcontroller.py | 170 ++---------------- .../toaster/bldcontrol/localhostbecontroller.py | 191 ++++++++++++++++++++ bitbake/lib/toaster/bldcontrol/sshbecontroller.py | 193 +++++++++++++++++++++ bitbake/lib/toaster/bldcontrol/tests.py | 116 ++++++++++--- 4 files changed, 488 insertions(+), 182 deletions(-) create mode 100644 bitbake/lib/toaster/bldcontrol/localhostbecontroller.py create mode 100644 bitbake/lib/toaster/bldcontrol/sshbecontroller.py diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py index bf9cdf9f67..6812ae3e6e 100644 --- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -26,10 +26,6 @@ import re from django.db import transaction from django.db.models import Q from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake -import subprocess - -from toastermain import settings - # load Bitbake components path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) @@ -72,6 +68,10 @@ def getBuildEnvironmentController(**kwargs): The return object MUST always be a BuildEnvironmentController. """ + + from localhostbecontroller import LocalhostBEController + from sshbecontroller import SSHBEController + be = BuildEnvironment.objects.filter(Q(**kwargs))[0] if be.betype == BuildEnvironment.TYPE_LOCAL: return LocalhostBEController(be) @@ -81,6 +81,13 @@ def getBuildEnvironmentController(**kwargs): raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) +def _getgitcheckoutdirectoryname(url): + """ Utility that returns the last component of a git path as directory + """ + import re + components = re.split(r'[:\.\/]', url) + return components[-2] if components[-1] == "git" else components[-1] + class BuildEnvironmentController(object): """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST @@ -110,6 +117,7 @@ class BuildEnvironmentController(object): self.be = be self.connection = None + def startBBServer(self): """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI. After this method executes, self.be bbaddress/bbport MUST point to a running and free server, @@ -173,157 +181,3 @@ class ShellCmdException(Exception): class BuildSetupException(Exception): pass -class LocalhostBEController(BuildEnvironmentController): - """ Implementation of the BuildEnvironmentController for the localhost; - this controller manages the default build directory, - the server setup and system start and stop for the localhost-type build environment - - """ - - def __init__(self, be): - super(LocalhostBEController, self).__init__(be) - self.dburl = settings.getDATABASE_URL() - self.pokydirname = None - - def _shellcmd(self, command, cwd = None): - if cwd is None: - cwd = self.be.sourcedir - - p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out,err) = p.communicate() - if p.returncode: - if len(err) == 0: - err = "command: %s \n%s" % (command, out) - else: - err = "command: %s \n%s" % (command, err) - raise ShellCmdException(err) - else: - return out - - def _createdirpath(self, path): - from os.path import dirname as DN - if not os.path.exists(DN(path)): - self._createdirpath(DN(path)) - if not os.path.exists(path): - os.mkdir(path, 0755) - - def _startBE(self): - assert self.pokydirname and os.path.exists(self.pokydirname) - self._createdirpath(self.be.builddir) - self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) - - def startBBServer(self): - assert self.pokydirname and os.path.exists(self.pokydirname) - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) - # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected - # but since they start async without any return, we just wait a bit - print "Started server" - assert self.be.sourcedir and os.path.exists(self.be.builddir) - self.be.bbaddress = "localhost" - self.be.bbport = "8200" - self.be.bbstate = BuildEnvironment.SERVER_STARTED - self.be.save() - - def stopBBServer(self): - assert self.be.sourcedir - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % - (self.be.sourcedir, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)())) - self.be.bbstate = BuildEnvironment.SERVER_STOPPED - self.be.save() - print "Stopped server" - - def setLayers(self, bitbakes, layers): - """ a word of attention: by convention, the first layer for any build will be poky! """ - - assert self.be.sourcedir is not None - assert len(bitbakes) == 1 - # set layers in the layersource - - # 1. get a list of repos, and map dirpaths for each layer - gitrepos = {} - gitrepos[bitbakes[0].giturl] = [] - gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - - for layer in layers: - # we don't process local URLs - if layer.giturl.startswith("file://"): - continue - if not layer.giturl in gitrepos: - gitrepos[layer.giturl] = [] - gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) - for giturl in gitrepos.keys(): - commitid = gitrepos[giturl][0][2] - for e in gitrepos[giturl]: - if commitid != e[2]: - raise BuildSetupException("More than one commit per git url, unsupported configuration") - - def _getgitdirectoryname(url): - import re - components = re.split(r'[:\.\/]', url) - return components[-2] if components[-1] == "git" else components[-1] - - layerlist = [] - - # 2. checkout the repositories - for giturl in gitrepos.keys(): - localdirname = os.path.join(self.be.sourcedir, _getgitdirectoryname(giturl)) - print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname - - # make sure our directory is a git repository - if os.path.exists(localdirname): - if not giturl in self._shellcmd("git remote -v", localdirname): - raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) - else: - self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) - # checkout the needed commit - commit = gitrepos[giturl][0][2] - - # branch magic name "HEAD" will inhibit checkout - if commit != "HEAD": - print "DEBUG: checking out commit ", commit, "to", localdirname - self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) - - # take the localdirname as poky dir if we can find the oe-init-build-env - if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): - print "DEBUG: selected poky dir name", localdirname - self.pokydirname = localdirname - - # verify our repositories - for name, dirpath, commit in gitrepos[giturl]: - localdirpath = os.path.join(localdirname, dirpath) - if not os.path.exists(localdirpath): - raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) - - if name != "bitbake": - layerlist.append(localdirpath) - - print "DEBUG: current layer list ", layerlist - - # 3. configure the build environment, so we have a conf/bblayers.conf - assert self.pokydirname is not None - self._startBE() - - # 4. update the bblayers.conf - bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") - if not os.path.exists(bblayerconf): - raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - - conflines = open(bblayerconf, "r").readlines() - - bblayerconffile = open(bblayerconf, "w") - for i in xrange(len(conflines)): - if conflines[i].startswith("# line added by toaster"): - i += 2 - else: - bblayerconffile.write(conflines[i]) - - bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") - bblayerconffile.close() - - return True - - def release(self): - assert self.be.sourcedir and os.path.exists(self.be.builddir) - import shutil - shutil.rmtree(os.path.join(self.be.sourcedir, "build")) - assert not os.path.exists(self.be.builddir) diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py new file mode 100644 index 0000000000..fe7fd81fb9 --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -0,0 +1,191 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import sys +import re +from django.db import transaction +from django.db.models import Q +from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake +import subprocess + +from toastermain import settings + +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname + +class LocalhostBEController(BuildEnvironmentController): + """ Implementation of the BuildEnvironmentController for the localhost; + this controller manages the default build directory, + the server setup and system start and stop for the localhost-type build environment + + """ + + def __init__(self, be): + super(LocalhostBEController, self).__init__(be) + self.dburl = settings.getDATABASE_URL() + self.pokydirname = None + self.islayerset = False + + def _shellcmd(self, command, cwd = None): + if cwd is None: + cwd = self.be.sourcedir + + p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (out,err) = p.communicate() + if p.returncode: + if len(err) == 0: + err = "command: %s \n%s" % (command, out) + else: + err = "command: %s \n%s" % (command, err) + raise ShellCmdException(err) + else: + return out + + def _createdirpath(self, path): + from os.path import dirname as DN + if path == "": + raise Exception("Invalid path creation specified.") + if not os.path.exists(DN(path)): + self._createdirpath(DN(path)) + if not os.path.exists(path): + os.mkdir(path, 0755) + + def _setupBE(self): + assert self.pokydirname and os.path.exists(self.pokydirname) + self._createdirpath(self.be.builddir) + self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) + + def startBBServer(self): + assert self.pokydirname and os.path.exists(self.pokydirname) + assert self.islayerset + print("DEBUG: executing ", "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) + print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) + # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected + # but since they start async without any return, we just wait a bit + print "Started server" + assert self.be.sourcedir and os.path.exists(self.be.builddir) + self.be.bbaddress = "localhost" + self.be.bbport = "8200" + self.be.bbstate = BuildEnvironment.SERVER_STARTED + self.be.save() + + def stopBBServer(self): + assert self.pokydirname and os.path.exists(self.pokydirname) + assert self.islayerset + print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % + (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)())) + self.be.bbstate = BuildEnvironment.SERVER_STOPPED + self.be.save() + print "Stopped server" + + def setLayers(self, bitbakes, layers): + """ a word of attention: by convention, the first layer for any build will be poky! """ + + assert self.be.sourcedir is not None + assert len(bitbakes) == 1 + # set layers in the layersource + + # 1. get a list of repos, and map dirpaths for each layer + gitrepos = {} + gitrepos[bitbakes[0].giturl] = [] + gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) + + for layer in layers: + # we don't process local URLs + if layer.giturl.startswith("file://"): + continue + if not layer.giturl in gitrepos: + gitrepos[layer.giturl] = [] + gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) + for giturl in gitrepos.keys(): + commitid = gitrepos[giturl][0][2] + for e in gitrepos[giturl]: + if commitid != e[2]: + raise BuildSetupException("More than one commit per git url, unsupported configuration") + + + layerlist = [] + + # 2. checkout the repositories + for giturl in gitrepos.keys(): + localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) + print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname + + # make sure our directory is a git repository + if os.path.exists(localdirname): + if not giturl in self._shellcmd("git remote -v", localdirname): + raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) + else: + self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) + # checkout the needed commit + commit = gitrepos[giturl][0][2] + + # branch magic name "HEAD" will inhibit checkout + if commit != "HEAD": + print "DEBUG: checking out commit ", commit, "to", localdirname + self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) + + # take the localdirname as poky dir if we can find the oe-init-build-env + if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): + print "DEBUG: selected poky dir name", localdirname + self.pokydirname = localdirname + + # verify our repositories + for name, dirpath, commit in gitrepos[giturl]: + localdirpath = os.path.join(localdirname, dirpath) + if not os.path.exists(localdirpath): + raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) + + if name != "bitbake": + layerlist.append(localdirpath) + + print "DEBUG: current layer list ", layerlist + + # 3. configure the build environment, so we have a conf/bblayers.conf + assert self.pokydirname is not None + self._setupBE() + + # 4. update the bblayers.conf + bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") + if not os.path.exists(bblayerconf): + raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) + + conflines = open(bblayerconf, "r").readlines() + + bblayerconffile = open(bblayerconf, "w") + for i in xrange(len(conflines)): + if conflines[i].startswith("# line added by toaster"): + i += 2 + else: + bblayerconffile.write(conflines[i]) + + bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") + bblayerconffile.close() + + self.islayerset = True + return True + + def release(self): + assert self.be.sourcedir and os.path.exists(self.be.builddir) + import shutil + shutil.rmtree(os.path.join(self.be.sourcedir, "build")) + assert not os.path.exists(self.be.builddir) diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py new file mode 100644 index 0000000000..64674953dc --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py @@ -0,0 +1,193 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import sys +import re +from django.db import transaction +from django.db.models import Q +from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake +import subprocess + +from toastermain import settings + +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname + +def DN(path): + return "/".join(path.split("/")[0:-1]) + +class SSHBEController(BuildEnvironmentController): + """ Implementation of the BuildEnvironmentController for the localhost; + this controller manages the default build directory, + the server setup and system start and stop for the localhost-type build environment + + """ + + def __init__(self, be): + super(SSHBEController, self).__init__(be) + self.dburl = settings.getDATABASE_URL() + self.pokydirname = None + self.islayerset = False + + def _shellcmd(self, command, cwd = None): + if cwd is None: + cwd = self.be.sourcedir + + p = subprocess.Popen("ssh %s 'cd %s && %s'" % (self.be.address, cwd, command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (out,err) = p.communicate() + if p.returncode: + if len(err) == 0: + err = "command: %s \n%s" % (command, out) + else: + err = "command: %s \n%s" % (command, err) + raise ShellCmdException(err) + else: + return out.strip() + + def _pathexists(self, path): + try: + self._shellcmd("test -e \"%s\"" % path) + return True + except ShellCmdException as e: + return False + + def _pathcreate(self, path): + self._shellcmd("mkdir -p \"%s\"" % path) + + def _setupBE(self): + assert self.pokydirname and self._pathexists(self.pokydirname) + self._pathcreate(self.be.builddir) + self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) + + def startBBServer(self): + assert self.pokydirname and self._pathexists(self.pokydirname) + assert self.islayerset + print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) + # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected + # but since they start async without any return, we just wait a bit + print "Started server" + assert self.be.sourcedir and self._pathexists(self.be.builddir) + self.be.bbaddress = self.be.address.split("@")[-1] + self.be.bbport = "8200" + self.be.bbstate = BuildEnvironment.SERVER_STARTED + self.be.save() + + def stopBBServer(self): + assert self.pokydirname and self._pathexists(self.pokydirname) + assert self.islayerset + print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % + (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)())) + self.be.bbstate = BuildEnvironment.SERVER_STOPPED + self.be.save() + print "Stopped server" + + def setLayers(self, bitbakes, layers): + """ a word of attention: by convention, the first layer for any build will be poky! """ + + assert self.be.sourcedir is not None + assert len(bitbakes) == 1 + # set layers in the layersource + + # 1. get a list of repos, and map dirpaths for each layer + gitrepos = {} + gitrepos[bitbakes[0].giturl] = [] + gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) + + for layer in layers: + # we don't process local URLs + if layer.giturl.startswith("file://"): + continue + if not layer.giturl in gitrepos: + gitrepos[layer.giturl] = [] + gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) + for giturl in gitrepos.keys(): + commitid = gitrepos[giturl][0][2] + for e in gitrepos[giturl]: + if commitid != e[2]: + raise BuildSetupException("More than one commit per git url, unsupported configuration") + + layerlist = [] + + # 2. checkout the repositories + for giturl in gitrepos.keys(): + import os + localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) + print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname + + # make sure our directory is a git repository + if self._pathexists(localdirname): + if not giturl in self._shellcmd("git remote -v", localdirname): + raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) + else: + self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) + # checkout the needed commit + commit = gitrepos[giturl][0][2] + + # branch magic name "HEAD" will inhibit checkout + if commit != "HEAD": + print "DEBUG: checking out commit ", commit, "to", localdirname + self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) + + # take the localdirname as poky dir if we can find the oe-init-build-env + if self.pokydirname is None and self._pathexists(os.path.join(localdirname, "oe-init-build-env")): + print "DEBUG: selected poky dir name", localdirname + self.pokydirname = localdirname + + # verify our repositories + for name, dirpath, commit in gitrepos[giturl]: + localdirpath = os.path.join(localdirname, dirpath) + if not self._pathexists(localdirpath): + raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) + + if name != "bitbake": + layerlist.append(localdirpath) + + print "DEBUG: current layer list ", layerlist + + # 3. configure the build environment, so we have a conf/bblayers.conf + assert self.pokydirname is not None + self._setupBE() + + # 4. update the bblayers.conf + bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") + if not self._pathexists(bblayerconf): + raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) + + conflines = open(bblayerconf, "r").readlines() + + bblayerconffile = open(bblayerconf, "w") + for i in xrange(len(conflines)): + if conflines[i].startswith("# line added by toaster"): + i += 2 + else: + bblayerconffile.write(conflines[i]) + + bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") + bblayerconffile.close() + + self.islayerset = True + return True + + def release(self): + assert self.be.sourcedir and self._pathexists(self.be.builddir) + import shutil + shutil.rmtree(os.path.join(self.be.sourcedir, "build")) + assert not self._pathexists(self.be.builddir) diff --git a/bitbake/lib/toaster/bldcontrol/tests.py b/bitbake/lib/toaster/bldcontrol/tests.py index ebe477d8a8..4577c3f03b 100644 --- a/bitbake/lib/toaster/bldcontrol/tests.py +++ b/bitbake/lib/toaster/bldcontrol/tests.py @@ -7,46 +7,114 @@ Replace this with more appropriate tests for your application. from django.test import TestCase -from bldcontrol.bbcontroller import LocalhostBEController, BitbakeController +from bldcontrol.bbcontroller import BitbakeController +from bldcontrol.localhostbecontroller import LocalhostBEController +from bldcontrol.sshbecontroller import SSHBEController from bldcontrol.models import BuildEnvironment, BuildRequest from bldcontrol.management.commands.runbuilds import Command import socket import subprocess -class LocalhostBEControllerTests(TestCase): - def test_StartAndStopServer(self): - obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL) - lbc = LocalhostBEController(obe) +# standard poky data hardcoded for testing +BITBAKE_LAYERS = [type('bitbake_info', (object,), { "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "", "commit": "HEAD"})] +POKY_LAYERS = [ + type('poky_info', (object,), { "name": "meta", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta", "commit": "HEAD"}), + type('poky_info', (object,), { "name": "meta-yocto", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto", "commit": "HEAD"}), + type('poky_info', (object,), { "name": "meta-yocto-bsp", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto-bsp", "commit": "HEAD"}), + ] + - # test start server and stop - self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Port already occupied") - lbc.startBBServer() - self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Server not answering") - lbc.stopBBServer() - self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Server not stopped") +# we have an abstract test class designed to ensure that the controllers use a single interface +# specific controller tests only need to override the _getBuildEnvironment() method - # clean up - import subprocess - out, err = subprocess.Popen("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() +class BEControllerTests(object): + def _serverForceStop(self, bc): + err = bc._shellcmd("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill") self.assertTrue(err == '', "bitbake server pid %s not stopped" % err) - obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL) - lbc = LocalhostBEController(obe) + def test_serverStartAndStop(self): + obe = self._getBuildEnvironment() + bc = self._getBEController(obe) + bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info + + hostname = self.test_address.split("@")[-1] - bbc = lbc.getBBController() + # test start server and stop + self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Port already occupied") + bc.startBBServer() + self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not answering") + + bc.stopBBServer() + self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not stopped") + + self._serverForceStop(bc) + + def test_getBBController(self): + obe = self._getBuildEnvironment() + bc = self._getBEController(obe) + bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info + + bbc = bc.getBBController() self.assertTrue(isinstance(bbc, BitbakeController)) - # test set variable + # test set variable, use no build marker -1 for BR value try: - bbc.setVariable + bbc.setVariable("TOASTER_BRBE", "%d:%d" % (-1, obe.pk)) except Exception as e : self.fail("setVariable raised %s", e) - lbc.stopBBServer() - out, err = subprocess.Popen("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - self.assertTrue(err == '', "bitbake server pid %s not stopped" % err) + bc.stopBBServer() + + self._serverForceStop(bc) + +class LocalhostBEControllerTests(TestCase, BEControllerTests): + def __init__(self, *args): + super(LocalhostBEControllerTests, self).__init__(*args) + # hardcoded for Alex's machine; since the localhost BE is machine-dependent, + # I found no good way to abstractize this + self.test_sourcedir = "/home/ddalex/ssd/yocto" + self.test_builddir = "/home/ddalex/ssd/yocto/build" + self.test_address = "localhost" + + def _getBuildEnvironment(self): + return BuildEnvironment.objects.create( + lock = BuildEnvironment.LOCK_FREE, + betype = BuildEnvironment.TYPE_LOCAL, + address = self.test_address, + sourcedir = self.test_sourcedir, + builddir = self.test_builddir ) + + def _getBEController(self, obe): + return LocalhostBEController(obe) + +class SSHBEControllerTests(TestCase, BEControllerTests): + def __init__(self, *args): + super(SSHBEControllerTests, self).__init__(*args) + self.test_address = "ddalex-desktop.local" + # hardcoded for ddalex-desktop.local machine; since the localhost BE is machine-dependent, + # I found no good way to abstractize this + self.test_sourcedir = "/home/ddalex/ssd/yocto" + self.test_builddir = "/home/ddalex/ssd/yocto/build" + + def _getBuildEnvironment(self): + return BuildEnvironment.objects.create( + lock = BuildEnvironment.LOCK_FREE, + betype = BuildEnvironment.TYPE_SSH, + address = self.test_address, + sourcedir = self.test_sourcedir, + builddir = self.test_builddir ) + + def _getBEController(self, obe): + return SSHBEController(obe) + + def test_pathExists(self): + obe = BuildEnvironment.objects.create(betype = BuildEnvironment.TYPE_SSH, address= self.test_address) + sbc = SSHBEController(obe) + self.assertTrue(sbc._pathexists("/")) + self.assertFalse(sbc._pathexists("/.deadbeef")) + self.assertTrue(sbc._pathexists(sbc._shellcmd("pwd"))) class RunBuildsCommandTests(TestCase): @@ -67,8 +135,8 @@ class RunBuildsCommandTests(TestCase): self.assertRaises(IndexError, command._selectBuildEnvironment) def test_br_select(self): - from orm.models import Project - p, created = Project.objects.get_or_create(pk=1) + from orm.models import Project, Release, BitbakeVersion + p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch="HEAD")[0])[0]) obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p) command = Command() br = command._selectBuildRequest() -- cgit v1.2.3