diff options
Diffstat (limited to 'Source/CTest/cmCTestMultiProcessHandler.cxx')
-rw-r--r-- | Source/CTest/cmCTestMultiProcessHandler.cxx | 691 |
1 files changed, 691 insertions, 0 deletions
diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx new file mode 100644 index 000000000..2cae80215 --- /dev/null +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -0,0 +1,691 @@ +/*============================================================================ + CMake - Cross Platform Makefile Generator + Copyright 2000-2009 Kitware, Inc., Insight Software Consortium + + Distributed under the OSI-approved BSD License (the "License"); + see accompanying file Copyright.txt for details. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the License for more information. +============================================================================*/ +#include "cmCTestMultiProcessHandler.h" +#include "cmProcess.h" +#include "cmStandardIncludes.h" +#include "cmCTest.h" +#include "cmSystemTools.h" +#include <stdlib.h> +#include <stack> +#include <float.h> + +class TestComparator +{ +public: + TestComparator(cmCTestMultiProcessHandler* handler) : Handler(handler) {} + ~TestComparator() {} + + // Sorts tests in descending order of cost + bool operator() (int index1, int index2) const + { + return Handler->Properties[index1]->Cost > + Handler->Properties[index2]->Cost; + } + +private: + cmCTestMultiProcessHandler* Handler; +}; + +cmCTestMultiProcessHandler::cmCTestMultiProcessHandler() +{ + this->ParallelLevel = 1; + this->Completed = 0; + this->RunningCount = 0; + this->StopTimePassed = false; +} + +cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() +{ +} + + // Set the tests +void +cmCTestMultiProcessHandler::SetTests(TestMap& tests, + PropertiesMap& properties) +{ + this->Tests = tests; + this->Properties = properties; + this->Total = this->Tests.size(); + // set test run map to false for all + for(TestMap::iterator i = this->Tests.begin(); + i != this->Tests.end(); ++i) + { + this->TestRunningMap[i->first] = false; + this->TestFinishMap[i->first] = false; + } + if(!this->CTest->GetShowOnly()) + { + this->ReadCostData(); + this->CreateTestCostList(); + } +} + + // Set the max number of tests that can be run at the same time. +void cmCTestMultiProcessHandler::SetParallelLevel(size_t level) +{ + this->ParallelLevel = level < 1 ? 1 : level; +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::RunTests() +{ + this->CheckResume(); + if(!this->CheckCycles()) + { + return; + } + this->TestHandler->SetMaxIndex(this->FindMaxIndex()); + this->StartNextTests(); + while(this->Tests.size() != 0) + { + if(this->StopTimePassed) + { + return; + } + this->CheckOutput(); + this->StartNextTests(); + } + // let all running tests finish + while(this->CheckOutput()) + { + } + this->MarkFinished(); + this->UpdateCostData(); +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::StartTestProcess(int test) +{ + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "test " << test << "\n"); + this->TestRunningMap[test] = true; // mark the test as running + // now remove the test itself + this->EraseTest(test); + this->RunningCount += GetProcessorsUsed(test); + + cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler); + testRun->SetIndex(test); + testRun->SetTestProperties(this->Properties[test]); + + std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory(); + cmSystemTools::ChangeDirectory(this->Properties[test]->Directory.c_str()); + + // Lock the resources we'll be using + this->LockResources(test); + + if(testRun->StartTest(this->Total)) + { + this->RunningTests.insert(testRun); + } + else if(testRun->IsStopTimePassed()) + { + this->StopTimePassed = true; + delete testRun; + return; + } + else + { + this->UnlockResources(test); + this->Completed++; + this->TestFinishMap[test] = true; + this->TestRunningMap[test] = false; + this->RunningCount -= GetProcessorsUsed(test); + testRun->EndTest(this->Completed, this->Total, false); + this->Failed->push_back(this->Properties[test]->Name); + delete testRun; + } + cmSystemTools::ChangeDirectory(current_dir.c_str()); +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::LockResources(int index) +{ + for(std::set<std::string>::iterator i = + this->Properties[index]->LockedResources.begin(); + i != this->Properties[index]->LockedResources.end(); ++i) + { + this->LockedResources.insert(*i); + } +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::UnlockResources(int index) +{ + for(std::set<std::string>::iterator i = + this->Properties[index]->LockedResources.begin(); + i != this->Properties[index]->LockedResources.end(); ++i) + { + this->LockedResources.erase(*i); + } +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::EraseTest(int test) +{ + this->Tests.erase(test); + this->SortedTests.erase( + std::find(this->SortedTests.begin(), this->SortedTests.end(), test)); +} + +//--------------------------------------------------------- +inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test) +{ + size_t processors = + static_cast<int>(this->Properties[test]->Processors); + //If this is set to run serially, it must run alone. + //Also, if processors setting is set higher than the -j + //setting, we default to using all of the process slots. + if(this->Properties[test]->RunSerial + || processors > this->ParallelLevel) + { + processors = this->ParallelLevel; + } + return processors; +} + +//--------------------------------------------------------- +bool cmCTestMultiProcessHandler::StartTest(int test) +{ + //Check for locked resources + for(std::set<std::string>::iterator i = + this->Properties[test]->LockedResources.begin(); + i != this->Properties[test]->LockedResources.end(); ++i) + { + if(this->LockedResources.find(*i) != this->LockedResources.end()) + { + return false; + } + } + + // copy the depend tests locally because when + // a test is finished it will be removed from the depend list + // and we don't want to be iterating a list while removing from it + TestSet depends = this->Tests[test]; + size_t totalDepends = depends.size(); + if(totalDepends) + { + for(TestSet::const_iterator i = depends.begin(); + i != depends.end(); ++i) + { + // if the test is not already running then start it + if(!this->TestRunningMap[*i]) + { + // this test might be finished, but since + // this is a copy of the depend map we might + // still have it + if(!this->TestFinishMap[*i]) + { + // only start one test in this function + return this->StartTest(*i); + } + else + { + // the depend has been and finished + totalDepends--; + } + } + } + } + // if there are no depends left then run this test + if(totalDepends == 0) + { + this->StartTestProcess(test); + return true; + } + // This test was not able to start because it is waiting + // on depends to run + return false; +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::StartNextTests() +{ + size_t numToStart = this->ParallelLevel - this->RunningCount; + if(numToStart == 0) + { + return; + } + + TestList copy = this->SortedTests; + for(TestList::iterator test = copy.begin(); test != copy.end(); ++test) + { + //in case this test has already been started due to dependency + if(this->TestRunningMap[*test] || this->TestFinishMap[*test]) + { + continue; + } + size_t processors = GetProcessorsUsed(*test); + if(processors > numToStart) + { + return; + } + if(this->StartTest(*test)) + { + if(this->StopTimePassed) + { + return; + } + numToStart -= processors; + } + if(numToStart == 0) + { + return; + } + } +} + +//--------------------------------------------------------- +bool cmCTestMultiProcessHandler::CheckOutput() +{ + // no more output we are done + if(this->RunningTests.size() == 0) + { + return false; + } + std::vector<cmCTestRunTest*> finished; + std::string out, err; + for(std::set<cmCTestRunTest*>::const_iterator i = this->RunningTests.begin(); + i != this->RunningTests.end(); ++i) + { + cmCTestRunTest* p = *i; + if(!p->CheckOutput()) + { + finished.push_back(p); + } + } + for( std::vector<cmCTestRunTest*>::iterator i = finished.begin(); + i != finished.end(); ++i) + { + this->Completed++; + cmCTestRunTest* p = *i; + int test = p->GetIndex(); + + if(p->EndTest(this->Completed, this->Total, true)) + { + this->Passed->push_back(p->GetTestProperties()->Name); + } + else + { + this->Failed->push_back(p->GetTestProperties()->Name); + } + for(TestMap::iterator j = this->Tests.begin(); + j != this->Tests.end(); ++j) + { + j->second.erase(test); + } + this->TestFinishMap[test] = true; + this->TestRunningMap[test] = false; + this->RunningTests.erase(p); + this->WriteCheckpoint(test); + this->UnlockResources(test); + this->RunningCount -= GetProcessorsUsed(test); + delete p; + } + return true; +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::UpdateCostData() +{ + std::string fname = this->CTest->GetCostDataFile(); + std::string tmpout = fname + ".tmp"; + std::fstream fout; + fout.open(tmpout.c_str(), std::ios::out); + + PropertiesMap temp = this->Properties; + + if(cmSystemTools::FileExists(fname.c_str())) + { + std::ifstream fin; + fin.open(fname.c_str()); + + std::string line; + while(std::getline(fin, line)) + { + if(line == "---") break; + std::vector<cmsys::String> parts = + cmSystemTools::SplitString(line.c_str(), ' '); + //Format: <name> <previous_runs> <avg_cost> + if(parts.size() < 3) break; + + std::string name = parts[0]; + int prev = atoi(parts[1].c_str()); + float cost = static_cast<float>(atof(parts[2].c_str())); + + int index = this->SearchByName(name); + if(index == -1) + { + // This test is not in memory. We just rewrite the entry + fout << name << " " << prev << " " << cost << "\n"; + } + else + { + // Update with our new average cost + fout << name << " " << this->Properties[index]->PreviousRuns << " " + << this->Properties[index]->Cost << "\n"; + temp.erase(index); + } + } + fin.close(); + cmSystemTools::RemoveFile(fname.c_str()); + } + + // Add all tests not previously listed in the file + for(PropertiesMap::iterator i = temp.begin(); i != temp.end(); ++i) + { + fout << i->second->Name << " " << i->second->PreviousRuns << " " + << i->second->Cost << "\n"; + } + + // Write list of failed tests + fout << "---\n"; + for(std::vector<cmStdString>::iterator i = this->Failed->begin(); + i != this->Failed->end(); ++i) + { + fout << i->c_str() << "\n"; + } + fout.close(); + cmSystemTools::RenameFile(tmpout.c_str(), fname.c_str()); +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::ReadCostData() +{ + std::string fname = this->CTest->GetCostDataFile(); + + if(cmSystemTools::FileExists(fname.c_str(), true)) + { + std::ifstream fin; + fin.open(fname.c_str()); + std::string line; + while(std::getline(fin, line)) + { + if(line == "---") break; + + std::vector<cmsys::String> parts = + cmSystemTools::SplitString(line.c_str(), ' '); + + // Probably an older version of the file, will be fixed next run + if(parts.size() < 3) + { + fin.close(); + return; + } + + std::string name = parts[0]; + int prev = atoi(parts[1].c_str()); + float cost = static_cast<float>(atof(parts[2].c_str())); + + int index = this->SearchByName(name); + if(index == -1) continue; + + this->Properties[index]->PreviousRuns = prev; + // When not running in parallel mode, don't use cost data + if(this->ParallelLevel > 1 && + this->Properties[index] && + this->Properties[index]->Cost == 0) + { + this->Properties[index]->Cost = cost; + } + } + // Next part of the file is the failed tests + while(std::getline(fin, line)) + { + if(line != "") + { + this->LastTestsFailed.push_back(line); + } + } + fin.close(); + } +} + +//--------------------------------------------------------- +int cmCTestMultiProcessHandler::SearchByName(std::string name) +{ + int index = -1; + + for(PropertiesMap::iterator i = this->Properties.begin(); + i != this->Properties.end(); ++i) + { + if(i->second->Name == name) + { + index = i->first; + } + } + return index; +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::CreateTestCostList() +{ + for(TestMap::iterator i = this->Tests.begin(); + i != this->Tests.end(); ++i) + { + SortedTests.push_back(i->first); + + //If the test failed last time, it should be run first, so max the cost. + //Only do this for parallel runs; in non-parallel runs, avoid clobbering + //the test's explicitly set cost. + if(this->ParallelLevel > 1 && + std::find(this->LastTestsFailed.begin(), this->LastTestsFailed.end(), + this->Properties[i->first]->Name) != this->LastTestsFailed.end()) + { + this->Properties[i->first]->Cost = FLT_MAX; + } + } + + TestComparator comp(this); + std::stable_sort(SortedTests.begin(), SortedTests.end(), comp); +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::WriteCheckpoint(int index) +{ + std::string fname = this->CTest->GetBinaryDir() + + "/Testing/Temporary/CTestCheckpoint.txt"; + std::fstream fout; + fout.open(fname.c_str(), std::ios::app | std::ios::out); + fout << index << "\n"; + fout.close(); +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::MarkFinished() +{ + std::string fname = this->CTest->GetBinaryDir() + + "/Testing/Temporary/CTestCheckpoint.txt"; + cmSystemTools::RemoveFile(fname.c_str()); +} + +//--------------------------------------------------------- +//For ShowOnly mode +void cmCTestMultiProcessHandler::PrintTestList() +{ + this->TestHandler->SetMaxIndex(this->FindMaxIndex()); + int count = 0; + + for (PropertiesMap::iterator it = this->Properties.begin(); + it != this->Properties.end(); ++it) + { + count++; + cmCTestTestHandler::cmCTestTestProperties& p = *it->second; + + //push working dir + std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory(); + cmSystemTools::ChangeDirectory(p.Directory.c_str()); + + cmCTestRunTest testRun(this->TestHandler); + testRun.SetIndex(p.Index); + testRun.SetTestProperties(&p); + testRun.ComputeArguments(); //logs the command in verbose mode + + if(p.Labels.size()) //print the labels + { + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Labels:"); + } + for(std::vector<std::string>::iterator label = p.Labels.begin(); + label != p.Labels.end(); ++label) + { + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, " " << *label); + } + if(p.Labels.size()) //print the labels + { + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl); + } + + if (this->TestHandler->MemCheck) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, " Memory Check"); + } + else + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, " Test"); + } + cmOStringStream indexStr; + indexStr << " #" << p.Index << ":"; + cmCTestLog(this->CTest, HANDLER_OUTPUT, + std::setw(3 + getNumWidth(this->TestHandler->GetMaxIndex())) + << indexStr.str().c_str()); + cmCTestLog(this->CTest, HANDLER_OUTPUT, " "); + cmCTestLog(this->CTest, HANDLER_OUTPUT, p.Name.c_str() << std::endl); + //pop working dir + cmSystemTools::ChangeDirectory(current_dir.c_str()); + } + + cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl << "Total Tests: " + << this->Total << std::endl); +} + +void cmCTestMultiProcessHandler::PrintLabels() +{ + std::set<std::string> allLabels; + for (PropertiesMap::iterator it = this->Properties.begin(); + it != this->Properties.end(); ++it) + { + cmCTestTestHandler::cmCTestTestProperties& p = *it->second; + allLabels.insert(p.Labels.begin(), p.Labels.end()); + } + + if(allLabels.size()) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, "All Labels:" << std::endl); + } + else + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, "No Labels Exist" << std::endl); + } + for(std::set<std::string>::iterator label = allLabels.begin(); + label != allLabels.end(); ++label) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, " " << *label << std::endl); + } +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::CheckResume() +{ + std::string fname = this->CTest->GetBinaryDir() + + "/Testing/Temporary/CTestCheckpoint.txt"; + if(this->CTest->GetFailover()) + { + if(cmSystemTools::FileExists(fname.c_str(), true)) + { + *this->TestHandler->LogFile << "Resuming previously interrupted test set" + << std::endl + << "----------------------------------------------------------" + << std::endl; + + std::ifstream fin; + fin.open(fname.c_str()); + std::string line; + while(std::getline(fin, line)) + { + int index = atoi(line.c_str()); + this->RemoveTest(index); + } + fin.close(); + } + } + else if(cmSystemTools::FileExists(fname.c_str(), true)) + { + cmSystemTools::RemoveFile(fname.c_str()); + } +} + +//--------------------------------------------------------- +void cmCTestMultiProcessHandler::RemoveTest(int index) +{ + this->EraseTest(index); + this->Properties.erase(index); + this->TestRunningMap[index] = false; + this->TestFinishMap[index] = true; + this->Completed++; +} + +//--------------------------------------------------------- +int cmCTestMultiProcessHandler::FindMaxIndex() +{ + int max = 0; + cmCTestMultiProcessHandler::TestMap::iterator i = this->Tests.begin(); + for(; i != this->Tests.end(); ++i) + { + if(i->first > max) + { + max = i->first; + } + } + return max; +} + +//Returns true if no cycles exist in the dependency graph +bool cmCTestMultiProcessHandler::CheckCycles() +{ + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + "Checking test dependency graph..." << std::endl); + for(TestMap::iterator it = this->Tests.begin(); + it != this->Tests.end(); ++it) + { + //DFS from each element to itself + int root = it->first; + std::set<int> visited; + std::stack<int> s; + s.push(root); + while(!s.empty()) + { + int test = s.top(); + s.pop(); + if(visited.insert(test).second) + { + for(TestSet::iterator d = this->Tests[test].begin(); + d != this->Tests[test].end(); ++d) + { + if(*d == root) + { + //cycle exists + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Error: a cycle exists in the test dependency graph " + "for the test \"" << this->Properties[root]->Name << + "\".\nPlease fix the cycle and run ctest again.\n"); + return false; + } + else + { + s.push(*d); + } + } + } + } + } + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + "Checking test dependency graph end" << std::endl); + return true; +} |