Update script to parse log format that contains test iteration

2 files changed, 16 insertions, 7 deletions

diff --git a/scripts/extract_seconds.py b/scripts/extract_seconds.py
index 9cef9f88..ea68e155 100755
--- a/scripts/extract_seconds.py
+++ b/scripts/extract_seconds.py
@@ -30,7 +30,7 @@ def extract_seconds(input_file, output_file):
         if not start_time_found and line.find('Solving') != -1:
             start_time_found = True
             start_datetime = extract_datetime_from_line(line, log_created_year)
-        if line.find(', loss = ') != -1:
+        if line.find('Iteration') != -1:
             dt = extract_datetime_from_line(line, log_created_year)
             elapsed_seconds = (dt - start_datetime).total_seconds()
             out.write('%f\n' % elapsed_seconds)
diff --git a/scripts/parselog.sh b/scripts/parselog.sh
index 76084eb8..8b7ce473 100755
--- a/scripts/parselog.sh
+++ b/scripts/parselog.sh
@@ -9,16 +9,21 @@ echo "Usage parselog.sh /path/to/your.log"
 exit
 fi
 LOG=`basename $1`
-# For extraction of time since this line constains the start time
-grep '] Solving ' $1 > aux.txt
-grep -B 2 'Test ' $1 >> aux.txt
+grep -B 1 'Test ' $1 > aux.txt
 grep 'Iteration ' aux.txt | sed  's/.*Iteration \([[:digit:]]*\).*/\1/g' > aux0.txt
 grep 'Test score #0' aux.txt | awk '{print $8}' > aux1.txt
 grep 'Test score #1' aux.txt | awk '{print $8}' > aux2.txt
-./extract_seconds.py aux.txt aux3.txt
+
+# Extracting elpased seconds
+# For extraction of time since this line constains the start time
+grep '] Solving ' $1 > aux3.txt
+grep 'Testing net' $1 >> aux3.txt
+./extract_seconds.py aux3.txt aux4.txt
+
+# Generating
 echo '# Iters Seconds TestAccuracy TestLoss'> $LOG.test
-paste aux0.txt aux3.txt aux1.txt aux2.txt | column -t >> $LOG.test
-rm aux.txt aux0.txt aux1.txt aux2.txt aux3.txt
+paste aux0.txt aux4.txt aux1.txt aux2.txt | column -t >> $LOG.test
+rm aux.txt aux0.txt aux1.txt aux2.txt aux3.txt aux4.txt
 
 # For extraction of time since this line constains the start time
 grep '] Solving ' $1 > aux.txt
@@ -26,7 +31,11 @@ grep ', loss = ' $1 >> aux.txt
 grep 'Iteration ' aux.txt | sed  's/.*Iteration \([[:digit:]]*\).*/\1/g' > aux0.txt
 grep ', loss = ' $1 | awk '{print $9}' > aux1.txt
 grep ', lr = ' $1 | awk '{print $9}' > aux2.txt
+
+# Extracting elpased seconds
 ./extract_seconds.py aux.txt aux3.txt
+
+# Generating
 echo '# Iters Seconds TrainingLoss LearningRate'> $LOG.train
 paste aux0.txt aux3.txt aux1.txt aux2.txt | column -t >> $LOG.train
 rm aux.txt aux0.txt aux1.txt aux2.txt  aux3.txt