diff options
| -rwxr-xr-x | data/create_mnist.sh | 12 | ||||
| -rwxr-xr-x | data/get_mnist.sh | 6 | ||||
| -rwxr-xr-x | scripts/extract_seconds.py | 2 | ||||
| -rwxr-xr-x | scripts/parselog.sh | 21 | ||||
| -rw-r--r-- | src/caffe/solver.cpp | 4 |
5 files changed, 32 insertions, 13 deletions
diff --git a/data/create_mnist.sh b/data/create_mnist.sh new file mode 100755 index 00000000..6a93d8f1 --- /dev/null +++ b/data/create_mnist.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env sh +# This script converts the mnist data into leveldb format. + +echo "Creating leveldb..." + +rm -rf mnist-train-leveldb +rm -rf mnist-test-leveldb + +../build/examples/convert_mnist_data.bin train-images-idx3-ubyte train-labels-idx1-ubyte mnist-train-leveldb +../build/examples/convert_mnist_data.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte mnist-test-leveldb + +echo "Done."
\ No newline at end of file diff --git a/data/get_mnist.sh b/data/get_mnist.sh index c807a7b9..d624dc02 100755 --- a/data/get_mnist.sh +++ b/data/get_mnist.sh @@ -15,9 +15,7 @@ gunzip train-labels-idx1-ubyte.gz gunzip t10k-images-idx3-ubyte.gz gunzip t10k-labels-idx1-ubyte.gz -echo "Creating leveldb..." - -../build/examples/convert_mnist_data.bin train-images-idx3-ubyte train-labels-idx1-ubyte mnist-train-leveldb -../build/examples/convert_mnist_data.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte mnist-test-leveldb +# Creation is split out because leveldb sometimes causes segfault +# and needs to be re-created. echo "Done." diff --git a/scripts/extract_seconds.py b/scripts/extract_seconds.py index 9cef9f88..ea68e155 100755 --- a/scripts/extract_seconds.py +++ b/scripts/extract_seconds.py @@ -30,7 +30,7 @@ def extract_seconds(input_file, output_file): if not start_time_found and line.find('Solving') != -1: start_time_found = True start_datetime = extract_datetime_from_line(line, log_created_year) - if line.find(', loss = ') != -1: + if line.find('Iteration') != -1: dt = extract_datetime_from_line(line, log_created_year) elapsed_seconds = (dt - start_datetime).total_seconds() out.write('%f\n' % elapsed_seconds) diff --git a/scripts/parselog.sh b/scripts/parselog.sh index 76084eb8..8b7ce473 100755 --- a/scripts/parselog.sh +++ b/scripts/parselog.sh @@ -9,16 +9,21 @@ echo "Usage parselog.sh /path/to/your.log" exit fi LOG=`basename $1` -# For extraction of time since this line constains the start time -grep '] Solving ' $1 > aux.txt -grep -B 2 'Test ' $1 >> aux.txt +grep -B 1 'Test ' $1 > aux.txt grep 'Iteration ' aux.txt | sed 's/.*Iteration \([[:digit:]]*\).*/\1/g' > aux0.txt grep 'Test score #0' aux.txt | awk '{print $8}' > aux1.txt grep 'Test score #1' aux.txt | awk '{print $8}' > aux2.txt -./extract_seconds.py aux.txt aux3.txt + +# Extracting elpased seconds +# For extraction of time since this line constains the start time +grep '] Solving ' $1 > aux3.txt +grep 'Testing net' $1 >> aux3.txt +./extract_seconds.py aux3.txt aux4.txt + +# Generating echo '# Iters Seconds TestAccuracy TestLoss'> $LOG.test -paste aux0.txt aux3.txt aux1.txt aux2.txt | column -t >> $LOG.test -rm aux.txt aux0.txt aux1.txt aux2.txt aux3.txt +paste aux0.txt aux4.txt aux1.txt aux2.txt | column -t >> $LOG.test +rm aux.txt aux0.txt aux1.txt aux2.txt aux3.txt aux4.txt # For extraction of time since this line constains the start time grep '] Solving ' $1 > aux.txt @@ -26,7 +31,11 @@ grep ', loss = ' $1 >> aux.txt grep 'Iteration ' aux.txt | sed 's/.*Iteration \([[:digit:]]*\).*/\1/g' > aux0.txt grep ', loss = ' $1 | awk '{print $9}' > aux1.txt grep ', lr = ' $1 | awk '{print $9}' > aux2.txt + +# Extracting elpased seconds ./extract_seconds.py aux.txt aux3.txt + +# Generating echo '# Iters Seconds TrainingLoss LearningRate'> $LOG.train paste aux0.txt aux3.txt aux1.txt aux2.txt | column -t >> $LOG.train rm aux.txt aux0.txt aux1.txt aux2.txt aux3.txt diff --git a/src/caffe/solver.cpp b/src/caffe/solver.cpp index ca8568a9..340bbe1d 100644 --- a/src/caffe/solver.cpp +++ b/src/caffe/solver.cpp @@ -84,7 +84,7 @@ void Solver<Dtype>::Solve(const char* resume_file) { template <typename Dtype> void Solver<Dtype>::Test() { - LOG(INFO) << "Testing net"; + LOG(INFO) << "Iteration " << iter_ << ", Testing net"; NetParameter net_param; net_->ToProto(&net_param); CHECK_NOTNULL(test_net_.get())->CopyTrainedLayersFrom(net_param); @@ -111,7 +111,7 @@ void Solver<Dtype>::Test() { } } for (int i = 0; i < test_score.size(); ++i) { - LOG(INFO) << "Iteration " << iter_ << ", Test score #" << i << ": " + LOG(INFO) << "Test score #" << i << ": " << test_score[i] / param_.test_iter(); } } |