Add runtime estimates for the CTS v2 integration

Bug: 22230380
Change-Id: I2e05765708fb7c952d48aba87ab1d7ead0bbd865

4 files changed, 277 insertions, 7 deletions

diff --git a/android/cts/AndroidTest.xml b/android/cts/AndroidTest.xml
index 54cc39596..fc9458ccf 100644
--- a/android/cts/AndroidTest.xml
+++ b/android/cts/AndroidTest.xml
@@ -28,6 +28,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="unspecified"/>
+		<option name="runtime-hint" value="24m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES2"/>
@@ -35,6 +36,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="unspecified"/>
+		<option name="runtime-hint" value="40m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES3"/>
@@ -42,6 +44,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="unspecified"/>
+		<option name="runtime-hint" value="1h15m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES3"/>
@@ -49,6 +52,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="0"/>
+		<option name="runtime-hint" value="7m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES3"/>
@@ -56,6 +60,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="90"/>
+		<option name="runtime-hint" value="7m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES3"/>
@@ -63,6 +68,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="180"/>
+		<option name="runtime-hint" value="7m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES3"/>
@@ -70,6 +76,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="270"/>
+		<option name="runtime-hint" value="7m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES3"/>
@@ -77,6 +84,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms4"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="unspecified"/>
+		<option name="runtime-hint" value="10m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES3"/>
@@ -84,6 +92,7 @@
 		<option name="deqp-gl-config-name" value="rgb565d0s0ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="unspecified"/>
+		<option name="runtime-hint" value="10m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES31"/>
@@ -91,6 +100,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="unspecified"/>
+		<option name="runtime-hint" value="7h30m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES31"/>
@@ -98,6 +108,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="0"/>
+		<option name="runtime-hint" value="1m30s"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES31"/>
@@ -105,6 +116,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="90"/>
+		<option name="runtime-hint" value="1m30s"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES31"/>
@@ -112,6 +124,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="180"/>
+		<option name="runtime-hint" value="1m30s"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES31"/>
@@ -119,6 +132,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="270"/>
+		<option name="runtime-hint" value="1m30s"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES31"/>
@@ -126,6 +140,7 @@
 		<option name="deqp-gl-config-name" value="rgba8888d24s8ms4"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="unspecified"/>
+		<option name="runtime-hint" value="2m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-GLES31"/>
@@ -133,9 +148,11 @@
 		<option name="deqp-gl-config-name" value="rgb565d0s0ms0"/>
 		<option name="deqp-surface-type" value="window"/>
 		<option name="deqp-screen-rotation" value="unspecified"/>
+		<option name="runtime-hint" value="1m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-VK"/>
 		<option name="deqp-caselist-file" value="vk-master.txt"/>
+		<option name="runtime-hint" value="3h45m"/>
 	</test>
 </configuration>
diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
index 855476f33..362e2748e 100644
--- a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
+++ b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
@@ -38,6 +38,7 @@ import com.android.tradefed.testtype.IBuildReceiver;
 import com.android.tradefed.testtype.IAbiReceiver;
 import com.android.tradefed.testtype.IDeviceTest;
 import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IRuntimeHintProvider;
 import com.android.tradefed.testtype.IShardableTest;
 import com.android.tradefed.testtype.ITestCollector;
 import com.android.tradefed.testtype.ITestFilterReceiver;
@@ -75,8 +76,8 @@ import java.util.concurrent.TimeUnit;
  */
 @OptionClass(alias="deqp-test-runner")
 public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
-        ITestFilterReceiver, IAbiReceiver, IShardableTest, ITestCollector {
-
+        ITestFilterReceiver, IAbiReceiver, IShardableTest, ITestCollector,
+        IRuntimeHintProvider {
     private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
     private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
     private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
@@ -125,6 +126,10 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
                     + "cases. All test run callbacks will be triggered, but test execution will "
                     + "not be actually carried out.")
     private boolean mCollectTestsOnly = false;
+    @Option(name = "runtime-hint",
+            isTimeVal = true,
+            description="The estimated config runtime. Defaults to 200ms x num tests.")
+    private long mRuntimeHint = -1;
 
     private Collection<TestIdentifier> mRemainingTests = null;
     private Map<TestIdentifier, Set<BatchRunConfiguration>> mTestInstances = null;
@@ -148,7 +153,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
     }
 
     private DeqpTestRunner(DeqpTestRunner optionTemplate,
-            Map<TestIdentifier, Set<BatchRunConfiguration>> tests) {
+                Map<TestIdentifier, Set<BatchRunConfiguration>> tests) {
         copyOptions(this, optionTemplate);
         mTestInstances = tests;
     }
@@ -1957,9 +1962,16 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
         for (String filter : mExcludeFilters) {
             CLog.d("Exclude: %s", filter);
         }
-        CLog.i("Num tests before filtering: %d", mTestInstances.size());
-        if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) {
+
+        long originalTestCount = mTestInstances.size();
+        CLog.i("Num tests before filtering: %d", originalTestCount);
+        if ((!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) && originalTestCount > 0) {
             filterTests(mTestInstances, mIncludeFilters, mExcludeFilters);
+
+            // Update runtime estimation hint.
+            if (mRuntimeHint != -1) {
+                mRuntimeHint = (mRuntimeHint * mTestInstances.size()) / originalTestCount;
+            }
         }
         CLog.i("Num tests after filtering: %d", mTestInstances.size());
     }
@@ -1971,8 +1983,9 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
         final Map<String, String> emptyMap = Collections.emptyMap();
         // If sharded, split() will load the tests.
-        if (mTestInstances == null)
+        if (mTestInstances == null) {
             loadTests();
+        }
 
         mRemainingTests = new LinkedList<>(mTestInstances.keySet());
 
@@ -2090,7 +2103,34 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
         }
         runners.add(new DeqpTestRunner(this, currentSet));
 
+        // Compute new runtime hints
+        long originalSize = iterationSet.size();
+        if (originalSize > 0) {
+            long fullRuntimeMs = getRuntimeHint();
+            for (IRemoteTest remote: runners) {
+                DeqpTestRunner runner = (DeqpTestRunner)remote;
+                long shardRuntime = (fullRuntimeMs * runner.mTestInstances.size()) / originalSize;
+                runner.mRuntimeHint = shardRuntime;
+            }
+        }
+
         CLog.i("Split deqp tests into %d shards", runners.size());
         return runners;
     }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public long getRuntimeHint() {
+        if (mRuntimeHint != -1) {
+            return mRuntimeHint;
+        }
+        if (mTestInstances == null) {
+            loadTests();
+        }
+        // Tests normally take something like ~100ms. Some take a
+        // second. Let's guess 200ms per test.
+        return 200 * mTestInstances.size();
+    }
 }
diff --git a/android/cts/runner/tests/run_tests.sh b/android/cts/runner/tests/run_tests.sh
index 9a6935a26..70a9ce76f 100755
--- a/android/cts/runner/tests/run_tests.sh
+++ b/android/cts/runner/tests/run_tests.sh
@@ -42,7 +42,7 @@ TF_CONSOLE=com.android.tradefed.command.Console
 ############### Run the cts tests ###############
 JARS="
     compatibility-host-util\
-    cts-tradefed_v2\
+    cts-tradefed\
     ddmlib-prebuilt\
     hosttestlib\
     CtsDeqpTestCases\
diff --git a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
index eb2e590cc..5e994bbbf 100644
--- a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
+++ b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
@@ -30,6 +30,7 @@ import com.android.tradefed.device.ITestDevice;
 import com.android.tradefed.result.ITestInvocationListener;
 import com.android.tradefed.testtype.IAbi;
 import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IRuntimeHintProvider;
 import com.android.tradefed.util.IRunUtil;
 import com.android.tradefed.util.RunInterruptedException;
 
@@ -1990,6 +1991,218 @@ public class DeqpTestRunnerTest extends TestCase {
         EasyMock.verify(mockDevice, mockIDevice);
     }
 
+    public void testRuntimeHint_optionSet() throws Exception {
+        /* MultiLineReceiver expects "\r\n" line ending. */
+        final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+                + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+                + "INSTRUMENTATION_CODE: 0\r\n";
+
+        final TestIdentifier[] testIds = {
+                new TestIdentifier("dEQP-GLES3.info", "vendor"),
+                new TestIdentifier("dEQP-GLES3.info", "renderer"),
+                new TestIdentifier("dEQP-GLES3.info", "version"),
+                new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
+                new TestIdentifier("dEQP-GLES3.info", "extensions"),
+                new TestIdentifier("dEQP-GLES3.info", "render_target")
+        };
+
+        final String[] testPaths = {
+                "dEQP-GLES3.info.vendor",
+                "dEQP-GLES3.info.renderer",
+                "dEQP-GLES3.info.version",
+                "dEQP-GLES3.info.shading_language_version",
+                "dEQP-GLES3.info.extensions",
+                "dEQP-GLES3.info.render_target"
+        };
+
+        final String testTrie
+                = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
+
+        ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+        ITestInvocationListener mockListener
+                = EasyMock.createStrictMock(ITestInvocationListener.class);
+        IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+        Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+        for (TestIdentifier id : testIds) {
+            tests.add(id);
+        }
+
+        DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+        OptionSetter setter = new OptionSetter(deqpTest);
+        final long runtimeMs = 123456;
+        setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs));
+        assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint());
+
+        // Try running the tests as well. The unit tests do not set the hint be default,
+        // so that case is covered.
+
+        int version = 3 << 16;
+        EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+                .andReturn(Integer.toString(version)).atLeastOnce();
+
+        EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+                .once();
+        EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+                EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+                .andReturn(null).once();
+
+        expectRenderConfigQuery(mockDevice, 3, 0);
+
+        String commandLine = String.format(
+                "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+                + "--deqp-screen-rotation=unspecified "
+                + "--deqp-surface-type=window "
+                + "--deqp-log-images=disable "
+                + "--deqp-watchdog=enable",
+                CASE_LIST_FILE_NAME);
+
+        runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
+
+        mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
+        EasyMock.expectLastCall().once();
+
+        for (int i = 0; i < testPaths.length; i++) {
+            mockListener.testStarted(EasyMock.eq(testIds[i]));
+            EasyMock.expectLastCall().once();
+
+            mockListener.testEnded(EasyMock.eq(testIds[i]),
+                    EasyMock.<Map<String, String>>notNull());
+
+            EasyMock.expectLastCall().once();
+        }
+
+        mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+        EasyMock.expectLastCall().once();
+
+        EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+                .once();
+
+        EasyMock.replay(mockDevice, mockIDevice);
+        EasyMock.replay(mockListener);
+
+        deqpTest.setDevice(mockDevice);
+        deqpTest.run(mockListener);
+
+        EasyMock.verify(mockListener);
+        EasyMock.verify(mockDevice, mockIDevice);
+    }
+
+    public void testRuntimeHint_optionSetSharded() throws Exception {
+        final int TEST_COUNT = 1237;
+        final int SHARD_SIZE = 1000;
+
+        ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
+        for (int i = 0; i < TEST_COUNT; i++) {
+            testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
+        }
+
+        DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds);
+        OptionSetter setter = new OptionSetter(deqpTest);
+        final long fullRuntimeMs = testIds.size()*100;
+        setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
+
+        ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split();
+        assertEquals("First shard's time not proportional to test count",
+                 (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT,
+                 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint());
+        assertEquals("Second shard's time not proportional to test count",
+                 (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT,
+                 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint());
+    }
+
+    public void testRuntimeHint_optionNotSet() throws Exception {
+        final TestIdentifier[] testIds = {
+                new TestIdentifier("dEQP-GLES3.info", "vendor"),
+                new TestIdentifier("dEQP-GLES3.info", "renderer"),
+                new TestIdentifier("dEQP-GLES3.info", "version"),
+                new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
+                new TestIdentifier("dEQP-GLES3.info", "extensions"),
+                new TestIdentifier("dEQP-GLES3.info", "render_target")
+        };
+        Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+        for (TestIdentifier id : testIds) {
+            tests.add(id);
+        }
+
+        DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+        long runtime = deqpTest.getRuntimeHint();
+        assertTrue("Runtime for tests must be positive", runtime > 0);
+        assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s
+    }
+
+
     private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
             final String output) throws Exception {
         String cmd = String.format(