summaryrefslogtreecommitdiff
path: root/go/src
diff options
context:
space:
mode:
Diffstat (limited to 'go/src')
-rw-r--r--go/src/regtest.go274
-rw-r--r--go/src/xdelta/rstream.go71
-rw-r--r--go/src/xdelta/run.go71
-rw-r--r--go/src/xdelta/test.go164
-rw-r--r--go/src/xdelta/tgroup.go97
5 files changed, 677 insertions, 0 deletions
diff --git a/go/src/regtest.go b/go/src/regtest.go
new file mode 100644
index 0000000..9d91f69
--- /dev/null
+++ b/go/src/regtest.go
@@ -0,0 +1,274 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "path"
+ "os"
+ "sort"
+ "time"
+
+ "xdelta"
+)
+
+const (
+ xdataset = "/volume/home/jmacd/src/testdata"
+ xcompare = "/volume/home/jmacd/src/xdelta-devel/xdelta3/build/x86_64-pc-linux-gnu-m64/xoff64/xdelta3"
+ xdelta3 = "/volume/home/jmacd/src/xdelta-64bithash/xdelta3/build/x86_64-pc-linux-gnu-m64/usize64/xoff64/xdelta3"
+ seed = 1422253499919909358
+)
+
+type Config struct {
+ srcbuf_size int64
+ window_size int64
+ blocksize int
+}
+
+func NewC() Config {
+ // TODO make these (and above) flags
+ return Config{1<<26, 1<<22, 1<<16}
+}
+
+func (c Config) smokeTest(t *xdelta.TestGroup, p xdelta.Program) {
+ target := "Hello world!"
+ source := "Hello world, nice to meet you!"
+
+ enc, err := t.Exec("encode", p, true, []string{"-e"})
+ if err != nil {
+ t.Panic(err)
+ }
+ dec, err := t.Exec("decode", p, true, []string{"-d"})
+ if err != nil {
+ t.Panic(err)
+ }
+
+ encodeout := t.Drain(enc.Stdout, "encode.stdout")
+ decodeout := t.Drain(dec.Stdout, "decode.stdout")
+
+ t.Empty(enc.Stderr, "encode")
+ t.Empty(dec.Stderr, "decode")
+
+ t.TestWrite("encode.stdin", enc.Stdin, []byte(target))
+ t.TestWrite("encode.srcin", enc.Srcin, []byte(source))
+
+ t.TestWrite("decode.stdin", dec.Stdin, <-encodeout)
+ t.TestWrite("decode.srcin", dec.Srcin, []byte(source))
+
+ if do := string(<-decodeout); do != target {
+ t.Panic(fmt.Errorf("It's not working! %s\n!=\n%s\n", do, target))
+ }
+ t.Wait(enc, dec)
+}
+
+type PairTest struct {
+ // Input
+ Config
+ program xdelta.Program
+ source, target string
+
+ // Output
+ TestOutput
+}
+
+type TestOutput struct {
+ encoded int64
+ encDuration time.Duration
+ decDuration time.Duration
+ encSysDuration time.Duration
+ decSysDuration time.Duration
+}
+
+func (to *TestOutput) Add(a TestOutput) {
+ to.encoded += a.encoded
+ to.encDuration += a.encDuration
+ to.decDuration += a.decDuration
+ to.encSysDuration += a.encSysDuration
+ to.decSysDuration += a.decSysDuration
+}
+
+func (to *TestOutput) String() string {
+ return fmt.Sprintf("SIZE: %v\tT: %v\tTSYS: %v\tDT: %v\tDTSYS: %v",
+ to.encoded, to.encDuration, to.encSysDuration, to.decDuration, to.encSysDuration)
+}
+
+// P is the test program, Q is the reference version.
+func (cfg Config) datasetTest(t *xdelta.TestGroup, p, q xdelta.Program) {
+ dir, err := os.Open(xdataset)
+ if err != nil {
+ t.Panic(err)
+ }
+ dents, err := dir.Readdir(-1)
+ if err != nil {
+ t.Panic(err)
+ }
+ paths := make([]string, len(dents))
+ var total int64
+ for i, d := range dents {
+ if !d.Mode().IsRegular() {
+ continue
+ }
+ paths[i] = fmt.Sprint(xdataset, "/", d.Name())
+ total += d.Size()
+ }
+ meansize := total / int64(len(dents))
+ largest := uint(20)
+ for ; largest <= 31 && 1<<largest < meansize; largest++ {}
+
+ sort.Strings(paths)
+
+ testSum := map[uint]*TestOutput{}
+ compSum := map[uint]*TestOutput{}
+
+ for _, in1 := range paths {
+ for _, in2 := range paths {
+ if in1 == in2 { continue }
+
+ // 1/4, 1/2, and 1 of the power-of-2 rounded-up mean size
+ for b := largest - 2; b <= largest; b++ {
+ if _, has := testSum[b]; !has {
+ testSum[b] = &TestOutput{}
+ compSum[b] = &TestOutput{}
+ }
+ c1 := cfg
+ c1.srcbuf_size = 1<<b
+ ptest := &PairTest{c1, p, in1, in2, TestOutput{-1, 0, 0, 0, 0}}
+ ptest.datasetPairTest(t, 1<<b);
+ qtest := &PairTest{c1, q, in1, in2, TestOutput{-1, 0, 0, 0, 0}}
+ qtest.datasetPairTest(t, 1<<b)
+
+ testSum[b].Add(ptest.TestOutput)
+ compSum[b].Add(qtest.TestOutput)
+
+ fmt.Printf("%s, %s: %.2f%% %+d/%d\n\tE:%.2f%%/%s(%.2f%%/%s) D:%.2f%%/%s(%.2f%%/%s) [B=%d]\n",
+ path.Base(in1), path.Base(in2),
+ float64(ptest.encoded - qtest.encoded) * 100.0 / float64(qtest.encoded),
+ ptest.encoded - qtest.encoded,
+ qtest.encoded,
+ (ptest.encDuration - qtest.encDuration).Seconds() * 100.0 / qtest.encDuration.Seconds(),
+ qtest.encDuration,
+ (ptest.decDuration - qtest.decDuration).Seconds() * 100.0 / qtest.decDuration.Seconds(),
+ qtest.encDuration,
+ (ptest.encSysDuration - qtest.encSysDuration).Seconds() * 100.0 / qtest.encSysDuration.Seconds(),
+ qtest.encSysDuration,
+ (ptest.decSysDuration - qtest.decSysDuration).Seconds() * 100.0 / qtest.decSysDuration.Seconds(),
+ qtest.decSysDuration,
+ 1<<b)
+ }
+ }
+ }
+ var keys []uint
+ for k, _ := range testSum {
+ keys = append(keys, k)
+ }
+ for _, k := range keys {
+ fmt.Printf("B=%v\nTEST: %v\nCOMP: %v\n", 1<<k, testSum[k], compSum[k])
+ }
+}
+
+func (pt *PairTest) datasetPairTest(t *xdelta.TestGroup, meanSize int64) {
+ cfg := pt.Config
+ eargs := []string{"-e", fmt.Sprint("-B", cfg.srcbuf_size), // "-q",
+ fmt.Sprint("-W", cfg.window_size), "-s", pt.source,
+ "-I0", "-S", "none", pt.target}
+ enc, err := t.Exec("encode", pt.program, false, eargs)
+ if err != nil {
+ t.Panic(err)
+ }
+
+ dargs := []string{"-dc", fmt.Sprint("-B", cfg.srcbuf_size), //"-q",
+ fmt.Sprint("-W", cfg.window_size), "-s", pt.source,
+ "-S", "none"}
+
+ dec, err := t.Exec("decode", pt.program, false, dargs)
+ if err != nil {
+ t.Panic(err)
+ }
+ tgt_check, err := os.Open(pt.target)
+ if err != nil {
+ t.Panic(err)
+ }
+ tgt_info, err := tgt_check.Stat()
+ if err != nil {
+ t.Panic(err)
+ }
+ t.Empty(enc.Stderr, "encode")
+ t.Empty(dec.Stderr, "decode")
+ t.CopyStreams(enc.Stdout, dec.Stdin, &pt.encoded)
+ t.CompareStreams(dec.Stdout, tgt_check, tgt_info.Size())
+
+ t.Wait(enc, dec)
+
+ pt.decDuration = dec.Cmd.ProcessState.UserTime()
+ pt.encDuration = enc.Cmd.ProcessState.UserTime()
+ pt.decSysDuration = dec.Cmd.ProcessState.SystemTime()
+ pt.encSysDuration = enc.Cmd.ProcessState.SystemTime()
+}
+
+func (cfg Config) offsetTest(t *xdelta.TestGroup, p xdelta.Program, offset, length int64) {
+ eargs := []string{"-e", "-0", fmt.Sprint("-B", cfg.srcbuf_size), "-q",
+ fmt.Sprint("-W", cfg.window_size)}
+ enc, err := t.Exec("encode", p, true, eargs)
+ if err != nil {
+ t.Panic(err)
+ }
+
+ dargs := []string{"-d", fmt.Sprint("-B", cfg.srcbuf_size), "-q",
+ fmt.Sprint("-W", cfg.window_size)}
+ dec, err := t.Exec("decode", p, true, dargs)
+ if err != nil {
+ t.Panic(err)
+ }
+
+ // The pipe used to read the decoder output and compare
+ // against the target.
+ read, write := io.Pipe()
+
+ t.Empty(enc.Stderr, "encode")
+ t.Empty(dec.Stderr, "decode")
+
+ var encoded_size int64
+ t.CopyStreams(enc.Stdout, dec.Stdin, &encoded_size)
+ t.CompareStreams(dec.Stdout, read, length)
+
+ // The decoder output ("read", above) is compared with the
+ // test-provided output ("write", below). The following
+ // generates two identical inputs.
+ t.WriteRstreams("encode", seed, offset, length, enc.Srcin, enc.Stdin)
+ t.WriteRstreams("decode", seed, offset, length, dec.Srcin, write)
+ t.Wait(enc, dec)
+
+ expect := cfg.srcbuf_size - offset
+ if float64(encoded_size) < (0.95 * float64(expect)) ||
+ float64(encoded_size) > (1.05 * float64(expect)) {
+ t.Fail("encoded size should be ~=", expect, ", actual ", encoded_size)
+ }
+}
+
+func main() {
+ r, err := xdelta.NewRunner()
+ if err != nil {
+ panic(err)
+ }
+ defer r.Cleanup()
+
+ cfg := NewC()
+
+ prog := xdelta.Program{xdelta3}
+
+ r.RunTest("smoketest", func(t *xdelta.TestGroup) { cfg.smokeTest(t, prog) })
+
+ for i := uint(29); i <= 33; i += 1 {
+ // The arguments to offsetTest are offset, source
+ // window size, and file size. The source window size
+ // is (2 << i) and (in the 3.0x release branch) is
+ // limited to 2^31, so the the greatest value of i is
+ // 30.
+ cfg.srcbuf_size = 2 << i
+ r.RunTest(fmt.Sprint("offset", i), func(t *xdelta.TestGroup) {
+ cfg.offsetTest(t, prog, 1 << i, 3 << i) })
+ }
+
+ comp := xdelta.Program{xcompare}
+
+ r.RunTest("dataset", func(t *xdelta.TestGroup) { cfg.datasetTest(t, prog, comp) })
+}
diff --git a/go/src/xdelta/rstream.go b/go/src/xdelta/rstream.go
new file mode 100644
index 0000000..99c3d17
--- /dev/null
+++ b/go/src/xdelta/rstream.go
@@ -0,0 +1,71 @@
+package xdelta
+
+
+import (
+ "io"
+ "math/rand"
+)
+
+const (
+ blocksize = 1<<17
+)
+
+func (t *TestGroup) WriteRstreams(desc string, seed, offset, len int64,
+ src, tgt io.WriteCloser) {
+ t.Go("src-write:"+desc, func (g *Goroutine) {
+ writeOne(g, seed, 0, len, tgt, false)
+ })
+ t.Go("tgt-write:"+desc, func (g *Goroutine) {
+ writeOne(g, seed, offset, len, src, true)
+ })
+}
+
+func writeOne(g *Goroutine, seed, offset, len int64, stream io.WriteCloser, readall bool) {
+ if !readall {
+ // Allow the source-read to fail or block until the process terminates.
+ // This behavior is reserved for the decoder, which is not required to
+ // read the entire source.
+ g.OK()
+ }
+ if offset != 0 {
+ // Fill with other random data until the offset
+ if err := writeRand(g, rand.New(rand.NewSource(^seed)), offset, stream); err != nil {
+ g.Panic(err)
+ }
+ }
+ if err := writeRand(g, rand.New(rand.NewSource(seed)),
+ len - offset, stream); err != nil {
+ g.Panic(err)
+ }
+ if err := stream.Close(); err != nil {
+ g.Panic(err)
+ }
+ g.OK()
+}
+
+func writeRand(g *Goroutine, r *rand.Rand, len int64, s io.Writer) error {
+ blk := make([]byte, blocksize)
+ for len > 0 {
+ fillRand(r, blk)
+ c := blocksize
+ if len < blocksize {
+ c = int(len)
+ }
+ if _, err := s.Write(blk[0:c]); err != nil {
+ return err
+ }
+ len -= int64(c)
+ }
+ return nil
+}
+
+func fillRand(r *rand.Rand, blk []byte) {
+ for p := 0; p < len(blk); {
+ v := r.Int63()
+ for i := 7; i != 0 && p < len(blk); i-- {
+ blk[p] = byte(v)
+ p++
+ v >>= 8
+ }
+ }
+}
diff --git a/go/src/xdelta/run.go b/go/src/xdelta/run.go
new file mode 100644
index 0000000..448fabe
--- /dev/null
+++ b/go/src/xdelta/run.go
@@ -0,0 +1,71 @@
+package xdelta
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+)
+
+type Program struct {
+ Path string
+}
+
+type Run struct {
+ Cmd exec.Cmd
+ Srcfile string
+ Stdin io.WriteCloser
+ Srcin io.WriteCloser
+ Stdout io.ReadCloser
+ Stderr io.ReadCloser
+}
+
+type Runner struct {
+ Testdir string
+}
+
+func (r *Run) Wait() error {
+ return r.Cmd.Wait()
+}
+
+func NewRunner() (*Runner, error) {
+ if dir, err := ioutil.TempDir(tmpDir, "xrt"); err != nil {
+ return nil, err
+ } else {
+ return &Runner{dir}, nil
+ }
+}
+
+func (r *Runner) newTestGroup(name string) (*TestGroup) {
+ tg := &TestGroup{Runner: r}
+ tg.WaitGroup.Add(1)
+ g0 := &Goroutine{tg, name, false}
+ tg.running = append(tg.running, g0)
+ tg.main = g0
+ return tg
+}
+
+func (r *Runner) Cleanup() {
+ os.RemoveAll(r.Testdir)
+}
+
+func (r *Runner) RunTest(name string, f func (t *TestGroup)) {
+ t := r.newTestGroup(name)
+ c := make(chan interface{})
+ go func() {
+ defer func() {
+ rec := recover()
+ c <- rec
+ }()
+ fmt.Println("Testing", name, "...")
+ f(t)
+ c <- nil
+ }()
+ rec := <- c
+ if t.errors == nil && rec == nil {
+ fmt.Println("Success:", name)
+ } else {
+ fmt.Println("FAILED:", name, t.errors, rec)
+ }
+}
diff --git a/go/src/xdelta/test.go b/go/src/xdelta/test.go
new file mode 100644
index 0000000..7210698
--- /dev/null
+++ b/go/src/xdelta/test.go
@@ -0,0 +1,164 @@
+package xdelta
+
+import (
+ "bufio"
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path"
+ "sync/atomic"
+
+ "golang.org/x/sys/unix"
+)
+
+var (
+ tmpDir = "/tmp"
+ srcSeq int64
+)
+
+func (t *TestGroup) Drain(f io.ReadCloser, desc string) <-chan []byte {
+ c := make(chan []byte)
+ t.Go(desc, func(g *Goroutine) {
+ if b, err := ioutil.ReadAll(f); err != nil {
+ g.Panic(err)
+ } else {
+ c <- b
+ }
+ g.OK()
+ })
+ return c
+}
+
+func (t *TestGroup) Empty(f io.ReadCloser, desc string) *Goroutine {
+ return t.Go("empty:"+desc, func (g *Goroutine) {
+ s := bufio.NewScanner(f)
+ for s.Scan() {
+ os.Stderr.Write([]byte(fmt.Sprint(desc, ": ", s.Text(), "\n")))
+ }
+ err := s.Err()
+ f.Close()
+ if err != nil {
+ g.Panic(err)
+ }
+ g.OK()
+ })
+}
+
+func (t *TestGroup) TestWrite(what string, f io.WriteCloser, b []byte) *Goroutine {
+ return t.Go("write", func(g *Goroutine) {
+ if _, err := f.Write(b); err != nil {
+ g.Panic(err)
+ }
+ if err := f.Close(); err != nil {
+ g.Panic(err)
+ }
+ g.OK()
+ })
+}
+
+func (t *TestGroup) CopyStreams(r io.ReadCloser, w io.WriteCloser, written *int64) *Goroutine {
+ return t.Go("copy", func(g *Goroutine) {
+ nwrite, err := io.Copy(w, r)
+ if err != nil {
+ g.Panic(err)
+ }
+ err = r.Close()
+ if err != nil {
+ g.Panic(err)
+ }
+ err = w.Close()
+ if err != nil {
+ g.Panic(err)
+ }
+ g.OK()
+ *written = nwrite
+ })
+}
+
+func (t *TestGroup) CompareStreams(r1 io.ReadCloser, r2 io.ReadCloser, length int64) *Goroutine {
+ return t.Go("compare", func(g *Goroutine) {
+ b1 := make([]byte, blocksize)
+ b2 := make([]byte, blocksize)
+ var idx int64
+ for length > 0 {
+ c := blocksize
+ if length < blocksize {
+ c = int(length)
+ }
+ if _, err := io.ReadFull(r1, b1[0:c]); err != nil {
+ g.Panic(err)
+ }
+ if _, err := io.ReadFull(r2, b2[0:c]); err != nil {
+ g.Panic(err)
+ }
+ if bytes.Compare(b1[0:c], b2[0:c]) != 0 {
+ fmt.Println("B1 is", string(b1[0:c]))
+ fmt.Println("B2 is", string(b2[0:c]))
+ g.Panic(errors.New(fmt.Sprint("Bytes do not compare at ", idx)))
+ }
+ length -= int64(c)
+ idx += int64(c)
+ }
+ g.OK()
+ })
+}
+
+func (t *TestGroup) Exec(desc string, p Program, srcfifo bool, flags []string) (*Run, error) {
+ var err error
+ run := &Run{}
+ args := []string{p.Path}
+ if srcfifo {
+ num := atomic.AddInt64(&srcSeq, 1)
+ run.Srcfile = path.Join(t.Runner.Testdir, fmt.Sprint("source", num))
+ if err = unix.Mkfifo(run.Srcfile, 0600); err != nil {
+ return nil, err
+ }
+ read, write := io.Pipe()
+ t.writeFifo(run.Srcfile, read)
+ run.Srcin = write
+ args = append(args, "-s")
+ args = append(args, run.Srcfile)
+ }
+ if run.Stdin, err = run.Cmd.StdinPipe(); err != nil {
+ return nil, err
+ }
+ if run.Stdout, err = run.Cmd.StdoutPipe(); err != nil {
+ return nil, err
+ }
+ if run.Stderr, err = run.Cmd.StderrPipe(); err != nil {
+ return nil, err
+ }
+
+ run.Cmd.Path = p.Path
+ run.Cmd.Args = append(args, flags...)
+ run.Cmd.Dir = t.Runner.Testdir
+ if serr := run.Cmd.Start(); serr != nil {
+ return nil, serr
+ }
+ return run, nil
+}
+
+func (t *TestGroup) Fail(v ...interface{}) {
+ panic(fmt.Sprintln(v...))
+}
+
+func (t *TestGroup) writeFifo(srcfile string, read io.Reader) *Goroutine {
+ return t.Go("compare", func(g *Goroutine) {
+ fifo, err := os.OpenFile(srcfile, os.O_WRONLY, 0600)
+ if err != nil {
+ fifo.Close()
+ g.Panic(err)
+ }
+ if _, err := io.Copy(fifo, read); err != nil {
+ fifo.Close()
+ g.Panic(err)
+ }
+ if err := fifo.Close(); err != nil {
+ g.Panic(err)
+ }
+ g.OK()
+ })
+}
diff --git a/go/src/xdelta/tgroup.go b/go/src/xdelta/tgroup.go
new file mode 100644
index 0000000..602b1e1
--- /dev/null
+++ b/go/src/xdelta/tgroup.go
@@ -0,0 +1,97 @@
+package xdelta
+
+import (
+ "fmt"
+ "runtime"
+ "sync"
+)
+
+type TestGroup struct {
+ *Runner
+ main *Goroutine
+ sync.Mutex
+ sync.WaitGroup
+ running []*Goroutine
+ errors []error
+ nonerrors []error // For tolerated / expected conditions
+}
+
+type Goroutine struct {
+ *TestGroup
+ name string
+ done bool
+}
+
+func (g *Goroutine) String() string {
+ return fmt.Sprint("[", g.name, "]")
+}
+
+func (g *Goroutine) finish(err error) {
+ wait := false
+ tg := g.TestGroup
+ sbuf := make([]byte, 4096)
+ sbuf = sbuf[0:runtime.Stack(sbuf, false)]
+ if err != nil {
+ err = fmt.Errorf("%v:%v:%v", g.name, err, string(sbuf))
+ }
+ tg.Lock()
+ if g.done {
+ if err != nil {
+ tg.nonerrors = append(tg.nonerrors, err)
+ }
+ } else {
+ wait = true
+ g.done = true
+ if err != nil {
+ tg.errors = append(tg.errors, err)
+ }
+ }
+ tg.Unlock()
+ if wait {
+ tg.WaitGroup.Done()
+ }
+}
+
+func (g *Goroutine) OK() {
+ g.finish(nil)
+}
+
+func (g *Goroutine) Panic(err error) {
+ g.finish(err)
+ if g != g.TestGroup.main {
+ runtime.Goexit()
+ }
+}
+
+func (t *TestGroup) Main() *Goroutine { return t.main }
+
+func (t *TestGroup) Panic(err error) { t.Main().Panic(err) }
+
+func (t *TestGroup) Go(name string, f func(*Goroutine)) *Goroutine {
+ g := &Goroutine{t, name, false}
+ t.Lock()
+ t.WaitGroup.Add(1)
+ t.running = append(t.running, g)
+ t.Unlock()
+ go f(g)
+ return g
+}
+
+func (t *TestGroup) Wait(procs... *Run) {
+ t.Main().OK()
+ t.WaitGroup.Wait()
+ for _, p := range procs {
+ if err := p.Wait(); err != nil {
+ t.errors = append(t.errors, err)
+ }
+ }
+ for _, err := range t.errors {
+ fmt.Println(":ERROR:", err)
+ }
+ for _, err := range t.nonerrors {
+ fmt.Println("(ERROR)", err)
+ }
+ if len(t.errors) != 0 {
+ t.Fail("Test failed with", len(t.errors), "errors")
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-26 15:09:57 +0000