Move common submission verification functions to scripts/verify

VulkanCTS submission verification functionality will be
signifcantly reused by OpenGL CTS.
It's nice to have commonly used functions in one place.

Change-Id: I61cd83556c6f961b50d1c27953dd57614c4f417a

3 files changed, 295 insertions, 0 deletions

diff --git a/scripts/verify/message.py b/scripts/verify/message.py
new file mode 100644
index 000000000..181bf35b8
--- /dev/null
+++ b/scripts/verify/message.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+#-------------------------------------------------------------------------
+# Vulkan CTS
+# ----------
+#
+# Copyright (c) 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#-------------------------------------------------------------------------
+
+import os
+
+class ValidationMessage:
+	TYPE_ERROR		= 0
+	TYPE_WARNING	= 1
+
+	def __init__ (self, type, filename, message):
+		self.type		= type
+		self.filename	= filename
+		self.message	= message
+
+	def __str__ (self):
+		prefix = {self.TYPE_ERROR: "ERROR: ", self.TYPE_WARNING: "WARNING: "}
+		return prefix[self.type] + os.path.basename(self.filename) + ": " + self.message
+
+def error (filename, message):
+	return ValidationMessage(ValidationMessage.TYPE_ERROR, filename, message)
+
+def warning (filename, message):
+	return ValidationMessage(ValidationMessage.TYPE_WARNING, filename, message)
diff --git a/scripts/verify/package.py b/scripts/verify/package.py
new file mode 100644
index 000000000..e178b4886
--- /dev/null
+++ b/scripts/verify/package.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+#-------------------------------------------------------------------------
+# Vulkan CTS
+# ----------
+#
+# Copyright (c) 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#-------------------------------------------------------------------------
+
+import os
+from fnmatch import fnmatch
+
+STATEMENT_PATTERN	= "STATEMENT-*"
+TEST_LOG_PATTERN	= "*.qpa"
+GIT_STATUS_PATTERN	= "*git-status.txt"
+GIT_LOG_PATTERN		= "*git-log.txt"
+PATCH_PATTERN		= "*.patch"
+
+class PackageDescription:
+	def __init__ (self, basePath, statement, testLogs, gitStatus, gitLog, patches, conformVersion, otherItems):
+		self.basePath		= basePath
+		self.statement		= statement
+		self.testLogs		= testLogs
+		self.gitStatus		= gitStatus
+		self.gitLog			= gitLog
+		self.patches		= patches
+		self.otherItems		= otherItems
+		self.conformVersion	= conformVersion
+
+def getPackageDescription (packagePath):
+	allItems	= os.listdir(packagePath)
+	statement	= None
+	testLogs	= []
+	gitStatus	= []
+	gitLog		= []
+	patches		= []
+	otherItems	= []
+	conformVersion	= None
+
+	for item in allItems:
+		if fnmatch(item, STATEMENT_PATTERN):
+			assert statement == None
+			statement = item
+		elif fnmatch(item, TEST_LOG_PATTERN):
+			testLogs.append(item)
+		elif fnmatch(item, GIT_STATUS_PATTERN):
+			gitStatus.append(item)
+		elif fnmatch(item, GIT_LOG_PATTERN):
+			gitLog.append(item)
+		elif fnmatch(item, PATCH_PATTERN):
+			patches.append(item)
+		else:
+			otherItems.append(item)
+
+	return PackageDescription(packagePath, statement, testLogs, gitStatus, gitLog, patches, conformVersion, otherItems)
diff --git a/scripts/verify/verify.py b/scripts/verify/verify.py
new file mode 100644
index 000000000..663bba589
--- /dev/null
+++ b/scripts/verify/verify.py
@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*-
+
+#-------------------------------------------------------------------------
+# Vulkan CTS
+# ----------
+#
+# Copyright (c) 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#-------------------------------------------------------------------------
+
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "log"))
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "build"))
+
+from common import readFile
+from message import *
+from log_parser import StatusCode, BatchResultParser
+
+ALLOWED_STATUS_CODES = set([
+		StatusCode.PASS,
+		StatusCode.NOT_SUPPORTED,
+		StatusCode.QUALITY_WARNING,
+		StatusCode.COMPATIBILITY_WARNING
+	])
+
+def readMustpass (filename):
+	f = open(filename, 'rb')
+	cases = []
+	for line in f:
+		s = line.strip()
+		if len(s) > 0:
+			cases.append(s)
+	return cases
+
+def readTestLog (filename):
+	parser = BatchResultParser()
+	return parser.parseFile(filename)
+
+def verifyTestLog (filename, mustpass):
+	results			= readTestLog(filename)
+	messages			= []
+	resultOrderOk	= True
+
+	# Mustpass case names must be unique
+	assert len(mustpass) == len(set(mustpass))
+
+	# Verify number of results
+	if len(results) != len(mustpass):
+		messages.append(error(filename, "Wrong number of test results, expected %d, found %d" % (len(mustpass), len(results))))
+
+	caseNameToResultNdx = {}
+	for ndx in xrange(len(results)):
+		result = results[ndx]
+		if not result in caseNameToResultNdx:
+			caseNameToResultNdx[result.name] = ndx
+		else:
+			messages.append(error(filename, "Multiple results for " + result.name))
+
+	# Verify that all results are present and valid
+	for ndx in xrange(len(mustpass)):
+		caseName = mustpass[ndx]
+
+		if caseName in caseNameToResultNdx:
+			resultNdx	= caseNameToResultNdx[caseName]
+			result		= results[resultNdx]
+
+			if resultNdx != ndx:
+				resultOrderOk = False
+
+			if not result.statusCode in ALLOWED_STATUS_CODES:
+				messages.append(error(filename, result.name + ": " + result.statusCode))
+		else:
+			messages.append(error(filename, "Missing result for " + caseName))
+
+	if len(results) == len(mustpass) and not resultOrderOk:
+		messages.append(error(filename, "Results are not in the expected order"))
+
+	return messages
+
+def beginsWith (str, prefix):
+	return str[:len(prefix)] == prefix
+
+def verifyStatement (package):
+	messages	= []
+
+	if package.statement != None:
+		statementPath	= os.path.join(package.basePath, package.statement)
+		statement		= readFile(statementPath)
+		hasVersion		= False
+		hasProduct		= False
+		hasCpu			= False
+		hasOs			= False
+
+		for line in statement.splitlines():
+			if beginsWith(line, "CONFORM_VERSION:"):
+				if hasVersion:
+					messages.append(error(statementPath, "Multiple CONFORM_VERSIONs"))
+				else:
+					hasVersion = True
+			elif beginsWith(line, "PRODUCT:"):
+				hasProduct = True # Multiple products allowed
+			elif beginsWith(line, "CPU:"):
+				if hasCpu:
+					messages.append(error(statementPath, "Multiple PRODUCTs"))
+				else:
+					hasCpu = True
+			elif beginsWith(line, "OS:"):
+				if hasOs:
+					messages.append(error(statementPath, "Multiple OSes"))
+				else:
+					hasOs = True
+
+		if not hasVersion:
+			messages.append(error(statementPath, "No CONFORM_VERSION"))
+		if not hasProduct:
+			messages.append(error(statementPath, "No PRODUCT"))
+		if not hasCpu:
+			messages.append(error(statementPath, "No CPU"))
+		if not hasOs:
+			messages.append(error(statementPath, "No OS"))
+	else:
+		messages.append(error(package.basePath, "Missing conformance statement file"))
+
+	return messages
+
+def verifyGitStatus (package):
+	messages = []
+
+	if len(package.gitStatus) > 0:
+		for s in package.gitStatus:
+			statusPath	= os.path.join(package.basePath, s)
+			status		= readFile(statusPath)
+
+			if status.find("nothing to commit, working directory clean") < 0 and status.find("nothing to commit, working tree clean") < 0:
+				messages.append(error(package.basePath, "Working directory is not clean"))
+	else:
+		messages.append(error(package.basePath, "Missing git status files"))
+
+	return messages
+
+def isGitLogEmpty (package, gitLog):
+	logPath	= os.path.join(package.basePath, gitLog)
+	log		= readFile(logPath)
+
+	return len(log.strip()) == 0
+
+def verifyGitLog (package):
+	messages = []
+
+	if len(package.gitLog) > 0:
+		for log in package.gitLog:
+			if not isGitLogEmpty(package, log):
+				messages.append(warning(os.path.join(package.basePath, log), "Log is not empty"))
+	else:
+		messages.append(error(package.basePath, "Missing git log files"))
+
+	return messages
+
+def verifyPatches (package):
+	messages	= []
+	hasPatches	= len(package.patches)
+	logEmpty	= True
+	for log in package.gitLog:
+		logEmpty &= isGitLogEmpty(package, log)
+
+	if hasPatches and logEmpty:
+		messages.append(error(package.basePath, "Package includes patches but log is empty"))
+	elif not hasPatches and not logEmpty:
+		messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches"))
+
+	return messages