Imported Upstream version 2.03upstream/2.03

7 files changed, 565 insertions, 0 deletions

diff --git a/util/check.sh b/util/check.sh
new file mode 100644
index 0000000..c1c2a2d
--- /dev/null
+++ b/util/check.sh
@@ -0,0 +1,43 @@
+#! /bin/sh
+set -e
+
+#
+# usage: util/check.sh [directory]
+#
+# This script runs lzotest with all algorithms on a complete directory tree.
+# It is not suitable for accurate timings.
+#
+# Copyright (C) 1996-2008 Markus Franz Xaver Johannes Oberhumer
+#
+
+LZOTEST="lzotest"
+test -x ./lzotest/lzotest && LZOTEST="./lzotest/lzotest"
+test -x ./lzotest.exe && LZOTEST="./lzotest.exe"
+test -x ./lzotest.out && LZOTEST="./lzotest.out"
+
+dir="${*-.}"
+
+TMPFILE="/tmp/lzo_$$.tmp"
+rm -f $TMPFILE
+(find $dir/ -type f -print > $TMPFILE) || true
+
+## methods=`$LZOTEST -m | sed -n 's/^ *-m\([0-9]*\).*/\1/p'`
+## methods="9721 9722 9723 9724 9725 9726 9727 9728 9729"
+methods="21 31 1 2 3 4 5 6 7 8 9 11 12 13 14 15 16 17 18 19 61 71 81"
+methods="$methods 111 112 115"
+methods="$methods 921 931 901 911"
+methods="$methods 902 912 942 962 972 982 992"
+##methods="71 972"
+##methods="1101 1102 1103 1104 1105 1106 1107"
+
+LFLAGS="-q -T -n2 -S"
+LFLAGS="-q -T -n2"
+
+for i in $methods; do
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS
+done
+
+rm -f $TMPFILE
+echo "Done."
+exit 0
+
diff --git a/util/checkasm.sh b/util/checkasm.sh
new file mode 100644
index 0000000..6e84f46
--- /dev/null
+++ b/util/checkasm.sh
@@ -0,0 +1,46 @@
+#! /bin/sh
+set -e
+
+#
+# usage: util/checkasm.sh [directory]
+#
+# This script runs lzotest with all assembler decompressors
+# on a complete directory tree.
+# It is not suitable for accurate timings.
+#
+# Copyright (C) 1996-2008 Markus Franz Xaver Johannes Oberhumer
+#
+
+LZOTEST="lzotest"
+test -x ./lzotest/lzotest && LZOTEST="./lzotest/lzotest"
+test -x ./lzotest.exe && LZOTEST="./lzotest.exe"
+test -x ./lzotest.out && LZOTEST="./lzotest.out"
+LFLAGS="-q"
+
+dir="${*-.}"
+
+TMPFILE="/tmp/lzo_$$.tmp"
+rm -f $TMPFILE
+(find $dir/ -type f -print > $TMPFILE) || true
+
+for i in 11; do
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -A
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -A -S
+done
+
+for i in 61; do
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -F
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -F -S
+done
+
+for i in 71 81; do
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -A
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -A -S
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -F
+    cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -F -S
+done
+
+rm -f $TMPFILE
+echo "Done."
+exit 0
+
diff --git a/util/notime.pl b/util/notime.pl
new file mode 100644
index 0000000..46eb459
--- /dev/null
+++ b/util/notime.pl
@@ -0,0 +1,26 @@
+#! /usr/bin/perl
+##
+## vi:ts=4
+##
+##---------------------------------------------------------------------------##
+##
+##  Author:
+##      Markus F.X.J. Oberhumer         <markus@oberhumer.com>
+##
+##  Description:
+##      Remove timing values from a table created by table.pl
+##
+##  Copyright (C) 1996-2008 Markus Franz Xaver Johannes Oberhumer
+##
+##---------------------------------------------------------------------------##
+
+
+while (<>) {
+    if (substr($_,56) =~ /^\s+[\d\.]+\s+[\d\.]+\s+\|\s*\n$/) {
+        substr($_,56) = "     0.000     0.000 |\n";
+    }
+    print;
+}
+
+exit(0);
+
diff --git a/util/overlap.sh b/util/overlap.sh
new file mode 100644
index 0000000..a90613d
--- /dev/null
+++ b/util/overlap.sh
@@ -0,0 +1,27 @@
+#! /bin/sh
+set -e
+
+#
+# usage: util/overlap.sh [directory]
+#
+# This script runs the overlap example program on a complete directory tree.
+#
+# Copyright (C) 1996-2008 Markus Franz Xaver Johannes Oberhumer
+#
+
+OVERLAP="overlap"
+test -x ./examples/overlap && OVERLAP="./examples/overlap"
+test -x ./overlap.exe && OVERLAP="./overlap.exe"
+
+dir="${*-.}"
+
+TMPFILE="/tmp/lzo_$$.tmp"
+rm -f $TMPFILE
+(find $dir/ -type f -print0 > $TMPFILE) || true
+
+cat $TMPFILE | xargs -0 -r $OVERLAP
+
+rm -f $TMPFILE
+echo "Done."
+exit 0
+
diff --git a/util/shortf.pl b/util/shortf.pl
new file mode 100644
index 0000000..0bee2e0
--- /dev/null
+++ b/util/shortf.pl
@@ -0,0 +1,31 @@
+#! /usr/bin/perl
+##
+## vi:ts=4
+##
+##---------------------------------------------------------------------------##
+##
+##  Author:
+##      Markus F.X.J. Oberhumer         <markus@oberhumer.com>
+##
+##  Description:
+##      Create short files for compression test
+##
+##  Copyright (C) 1996-2008 Markus Franz Xaver Johannes Oberhumer
+##
+##---------------------------------------------------------------------------##
+
+$c = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+$c = "\x00\x01\x02";
+$c = "\x00";
+
+$x = $c x 1024;
+
+for $i (0 .. 50) {
+    $name = sprintf("f%04d.dat",$i);
+    open(OUTFILE,">$name") || die "$0: cannot open '$name': $!";
+    binmode(OUTFILE);
+    print OUTFILE substr($x,0,$i);
+    close(OUTFILE);
+}
+
+exit(0);
diff --git a/util/table.pl b/util/table.pl
new file mode 100644
index 0000000..b4e1ac8
--- /dev/null
+++ b/util/table.pl
@@ -0,0 +1,360 @@
+#! /usr/bin/perl
+##
+## vi:ts=4:et
+##
+##---------------------------------------------------------------------------##
+##
+##  Author:
+##      Markus F.X.J. Oberhumer         <markus@oberhumer.com>
+##
+##  Description:
+##      Convert the output of the LZO lzotest program into a nice table.
+##
+##  Copyright (C) 1996-2008 Markus Franz Xaver Johannes Oberhumer
+##
+##---------------------------------------------------------------------------##
+
+$PROG = $0;
+require 'ctime.pl';
+
+#
+# get options
+#
+
+while ($_ = $ARGV[ $[ ], /^-/) {
+    shift(@ARGV);
+    /^--$/ && ($opt_last = 1, last);
+
+    /^--sort=name/ && ($opt_sort_summary_by_name++, next);
+    /^--sort=ratio/ && ($opt_sort_summary_by_ratio++, next);
+    /^-s/ && ($opt_summary_only++, next);
+    /^-t/ && ($opt_clear_time++, next);
+}
+
+
+$alg = '';
+$sep = "+" . ("-" x 76) . "+\n";
+
+$block_size = -1;
+
+$n = 0;
+@algs = ();
+%average = ();
+%total = ();
+
+$lzo_version_string = '';
+$lzo_version_date = '';
+
+
+# /***********************************************************************
+# //
+# ************************************************************************/
+
+while (<>) {
+
+    if (/(^|\s)(\d+)\s+block\-size/i) {
+        if ($block_size < 0) {
+            $block_size = $2;
+            &intro($block_size);
+        } elsif ($block_size != $2) {
+            die "$PROG: block-size: $block_size != $2\n";
+        }
+        next;
+    }
+
+    if (/^\s*LZO\s.*library\s+\(v\s*([\w\.\s]+)\s*\,\s*([^\)]+)\)/) {
+        $lzo_version_string = $1;
+        $lzo_version_date = $2;
+        next;
+    }
+
+    if (/^\s*(\S+(\s+\[\S+\])?)\s*(\|.*\|)\s*$/i) {
+        if ($1 ne $alg) {
+            &footer($1);
+            &header($1);
+        }
+        $line = $3;
+        &stats(*line);
+        print "$line\n" if (!$opt_summary_only);
+    }
+}
+&footer($1);
+
+&summary();
+
+exit(0);
+
+
+# /***********************************************************************
+# //
+# ************************************************************************/
+
+sub stats {
+    local (*l) = @_;
+    local ($x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8);
+
+    if ($l !~ /^\|\s*(.+?)\s+(\d+)\s+(\d+)\s+(\d+)\s+([\d\.]+\s+)?([\d\.]+\s+)?([\d\.]+)\s+([\d\.]+)\s*\|/) {
+        die $_;
+    }
+
+    $n++;
+
+    $x1 = $1; $x2 = $2; $x3 = $3; $x4 = $4;
+    $x5 = ($x2 > 0) ? $x4 * 100.0 / $x2 : 0.0;
+    $x6 = ($x2 > 0) ? $x4 *   8.0 / $x2 : 0.0;
+    $x7 = $7; $x8 = $8;
+
+    # convert from kB/s to MB/s (for old versions of lzotest)
+    if ($x7 =~ /\.\d\d$/) { $x7 = $x7 / 1000.0; }
+    if ($x8 =~ /\.\d\d$/) { $x8 = $x8 / 1000.0; }
+
+    if ($opt_clear_time) {
+        $x7 = $x8 = 0.0;
+    }
+
+    $s[0] += $x2;
+    $s[1] += $x3;
+    $s[2] += $x4;
+    $s[3] += $x5;
+    $s[4] += $x6;
+    if ($x7 > 0) {
+        $s[5] += 1.0 / $x7; $sn[5] += 1;
+    }
+    if ($x8 > 0) {
+        $s[6] += 1.0/ $x8; $sn[6] += 1;
+    }
+
+    $x1 =~ s/\s+$//;
+    $l = sprintf("| %-14s %10d %5d %9d %6.1f %5.2f %9.3f %9.3f |",
+                    $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8);
+}
+
+
+# /***********************************************************************
+# //
+# ************************************************************************/
+
+sub header {
+    local ($t) = @_;
+
+    $alg = $t;
+
+    # reset stats
+    $n = 0;
+    @s = (0, 0, 0, 0.0, 0.0, 0.0, 0.0);
+    @sn = (0, 0, 0, 0, 0, 0, 0);
+
+    return if $opt_summary_only;
+
+    print "\n$alg\n\n";
+    print $sep;
+print <<EndOfString;
+| File Name          Length   CxB    ComLen  Ratio% Bits  Com MB/s  Dec MB/s |
+| ---------          ------   ---    ------  -----  ----  --------  -------- |
+EndOfString
+}
+
+
+# /***********************************************************************
+# //
+# ************************************************************************/
+
+sub footer {
+    local ($t) = @_;
+    local ($shm5, $shm6);
+
+    return unless $alg;
+    die if $n <= 0;
+    die if $s[0] <= 0;
+
+    # harmonic mean
+    $shm5 = $s[5] > 0 ? $sn[5] / $s[5] : 0.0;
+    $shm6 = $s[6] > 0 ? $sn[6] / $s[6] : 0.0;
+
+    push(@algs,$alg);
+
+    $average{$alg} =
+        sprintf("| %-14s %10d %5d %9d %6.1f %5.2f %9.3f %9.3f |\n",
+            "Average", $s[0]/$n, $s[1]/$n, $s[2]/$n,
+            $s[3]/$n, $s[4]/$n,
+            $shm5, $shm6);
+
+    $total{$alg} =
+        sprintf("| %-14s %10d %5d %9d %6.1f %5.2f %9.3f %9.3f |\n",
+            "Total", $s[0], $s[1], $s[2],
+            $s[2]/$s[0]*100, $s[2]/$s[0]*8,
+            $shm5, $shm6);
+
+    return if $opt_summary_only;
+
+    print $sep;
+    print $average{$alg};
+    print $total{$alg};
+    print $sep, "\n";
+}
+
+
+# /***********************************************************************
+# //
+# ************************************************************************/
+
+$sort_mode = 0;
+
+sub cmp_by_ratio {
+    local ($aa, $bb);
+
+    if ($sort_mode == 0) {
+        $aa = $average{$a};
+        $bb = $average{$b};
+    } elsif ($sort_mode == 1) {
+        $aa = $total{$a};
+        $bb = $total{$b};
+    } else {
+        die;
+    }
+
+    ($aa =~ m%^\s*\|\s+\S+\s+\d+\s+\d+\s+\d+\s+(\S+)%) || die;
+    $aa = $1;
+    ($bb =~ m%^\s*\|\s+\S+\s+\d+\s+\d+\s+\d+\s+(\S+)%) || die;
+    $bb = $1;
+
+    # $aa < $bb;
+    $aa cmp $bb;
+}
+
+
+# /***********************************************************************
+# //
+# ************************************************************************/
+
+sub summary {
+    local ($l);
+    local (@k);
+
+    $sort_mode = 0;
+    if ($opt_sort_summary_by_name) {
+        @k = sort(@algs);
+    } elsif ($opt_sort_summary_by_ratio) {
+        @k = sort(cmp_by_ratio @algs);
+    } else {
+        @k = @algs;
+    }
+
+    print "\n\n";
+    print "Summary of average values\n\n";
+    print $sep;
+print <<EndOfString;
+| Algorithm          Length   CxB    ComLen  Ratio% Bits  Com MB/s  Dec MB/s |
+| ---------          ------   ---    ------  -----  ----  --------  -------- |
+EndOfString
+
+    for (@k) {
+        $l = $average{$_};
+        $l =~ s/Average[\s]{7}/sprintf("%-14s",$_)/e;
+        print $l;
+    }
+    print $sep;
+
+
+
+    $sort_mode = 1;
+    if ($opt_sort_summary_by_name) {
+        @k = sort(@algs);
+    } elsif ($opt_sort_summary_by_ratio) {
+        @k = sort(cmp_by_ratio @algs);
+    } else {
+        @k = @algs;
+    }
+
+    print "\n\n";
+    print "Summary of total values\n\n";
+    print $sep;
+print <<EndOfString;
+| Algorithm          Length   CxB    ComLen  Ratio% Bits  Com MB/s  Dec MB/s |
+| ---------          ------   ---    ------  -----  ----  --------  -------- |
+EndOfString
+
+    for (@k) {
+        $l = $total{$_};
+        $l =~ s/Total[\s]{9}/sprintf("%-14s",$_)/e;
+        print $l;
+    }
+    print $sep;
+}
+
+
+# /***********************************************************************
+# //
+# ************************************************************************/
+
+sub intro {
+    local ($bs) = @_;
+    local ($v, $t, $x);
+    local ($u, $uname_m, $uname_s, $uname_r);
+
+    $t = &ctime(time); chop($t);
+    $t = sprintf("%-55s |", $t);
+
+    $v='';
+    if ($lzo_version_string) {
+        $v = $lzo_version_string;
+        $v .= ', ' . $lzo_version_date if $lzo_version_date;
+        $v = sprintf("%-55s |", $v);
+        $v = sprintf("| LZO version      : %s\n", $v);
+    }
+
+    if ($bs % 1024 == 0) {
+        $x = sprintf("%d (= %d kB)", $bs, $bs / 1024);
+    } else {
+        $x = sprintf("%d (= %.3f kB)", $bs, $bs / 1024.0);
+    }
+    $x = sprintf("%-55s |", $x);
+
+    $u='';
+    if (1 == 1) {
+        $uname_s = `uname -s`; $uname_s =~ s/^\s+//; $uname_s =~ s/\s+$//;
+        $uname_r = `uname -r`; $uname_r =~ s/^\s+//; $uname_r =~ s/\s+$//;
+        $uname_m = `uname -m`; $uname_m =~ s/^\s+//; $uname_m =~ s/\s+$//;
+        if ($uname_s && $uname_m) {
+            $u = $uname_s;
+            $u .= ' ' . $uname_r if $uname_r;
+            $u .= ' ' . $uname_m;
+            $u = sprintf("%-55s |", $u);
+            $u = sprintf("| Operating system : %s\n", $u);
+        }
+    }
+    print <<EndOfString;
+
++----------------------------------------------------------------------------+
+| DATA COMPRESSION TEST                                                      |
+| =====================                                                      |
+| Time of run      : $t
+$v$u| Context length   : $x
++----------------------------------------------------------------------------+
+
+
+Notes:
+- CxB is the number of independent blocks a file was splitted
+- MB/s is the speed measured in 1,000,000 uncompressed bytes per second
+- all averages are calculated from the un-rounded values
+- the average ratio & bits are calculated by the arithmetic mean
+- the average speed is calculated by the harmonic mean
+
+
+EndOfString
+}
+
+__END__
+
+
+### insert something like this after 'Time of run':
+
+| Hardware         : Intel Pentium 133, 64 MB RAM, 256 kB Cache          |
+| Operating system : MS-DOS 7.10, HIMEM.SYS 3.95, DOS/4GW 1.97           |
+| Compiler         : Watcom C32 10.5                                     |
+| Compiler flags   : -mf -5r -oneatx                                     |
+| Test suite       : Calgary Corpus Suite                                |
+| Files in suite   : 14                                                  |
+| Timing accuracy  : One part in 100                                     |
+
+
diff --git a/util/uncompr.pl b/util/uncompr.pl
new file mode 100644
index 0000000..86c5265
--- /dev/null
+++ b/util/uncompr.pl
@@ -0,0 +1,32 @@
+#! /usr/bin/perl
+##
+## vi:ts=4
+##
+##---------------------------------------------------------------------------##
+##
+##  Author:
+##      Markus F.X.J. Oberhumer         <markus@oberhumer.com>
+##
+##  Description:
+##      Create uncompressible files for compression test
+##
+##  Copyright (C) 1996-2008 Markus Franz Xaver Johannes Oberhumer
+##
+##---------------------------------------------------------------------------##
+
+$x = ' ' x 65536;
+$i = 0;
+while ($i < 65536) {
+    substr($x,$i,1) = pack('C',rand(256));
+    $i++;
+}
+
+for $i (1,2,4,8,16,32,64) {
+    $name = sprintf("u%04d.dat",$i);
+    open(OUTFILE,">$name") || die "$0: cannot open '$name': $!";
+    binmode(OUTFILE);
+    print OUTFILE substr($x,0,$i*1024);
+    close(OUTFILE);
+}
+
+exit(0);