gzip_test.go

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sgzip

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"reflect"
	"strconv"
	"sync"
	"testing"
	"time"
)

// TestEmpty tests that an empty payload still forms a valid GZIP stream.
func TestEmpty(t *testing.T) {
	buf := new(bytes.Buffer)

	if err := NewWriter(buf).Close(); err != nil {
		t.Fatalf("Writer.Close: %v", err)
	}

	r, err := NewReader(buf)
	if err != nil {
		t.Fatalf("NewReader: %v", err)
	}
	b, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatalf("ReadAll: %v", err)
	}
	if len(b) != 0 {
		t.Fatalf("got %d bytes, want 0", len(b))
	}
	if err := r.Close(); err != nil {
		t.Fatalf("Reader.Close: %v", err)
	}
}

func TestEmptyBlockData(t *testing.T) {
	want := []uint32{10, 7}
	buf := new(bytes.Buffer)

	w := NewWriter(buf)
	if err := w.Close(); err != nil {
		t.Fatalf("Writer.Close: %v", err)
	}

	bd := w.BlockData()
	if !reflect.DeepEqual(bd, want) {
		t.Fatalf("want %+v, got: %+v", want, bd)
	}
}

// TestRoundTrip tests that gzipping and then gunzipping is the identity
// function.
func TestRoundTrip(t *testing.T) {
	buf := new(bytes.Buffer)

	w := NewWriter(buf)
	w.Comment = "comment"
	w.Extra = []byte("extra")
	w.ModTime = time.Unix(1e8, 0)
	w.Name = "name"
	if _, err := w.Write([]byte("payload")); err != nil {
		t.Fatalf("Write: %v", err)
	}
	if err := w.Close(); err != nil {
		t.Fatalf("Writer.Close: %v", err)
	}

	r, err := NewReader(buf)
	if err != nil {
		t.Fatalf("NewReader: %v", err)
	}
	b, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatalf("ReadAll: %v", err)
	}
	if string(b) != "payload" {
		t.Fatalf("payload is %q, want %q", string(b), "payload")
	}
	if r.Comment != "comment" {
		t.Fatalf("comment is %q, want %q", r.Comment, "comment")
	}
	if string(r.Extra) != "extra" {
		t.Fatalf("extra is %q, want %q", r.Extra, "extra")
	}
	if r.ModTime.Unix() != 1e8 {
		t.Fatalf("mtime is %d, want %d", r.ModTime.Unix(), uint32(1e8))
	}
	if r.Name != "name" {
		t.Fatalf("name is %q, want %q", r.Name, "name")
	}
	if err := r.Close(); err != nil {
		t.Fatalf("Reader.Close: %v", err)
	}
}

// TestLatin1 tests the internal functions for converting to and from Latin-1.
func TestLatin1(t *testing.T) {
	latin1 := []byte{0xc4, 'u', 0xdf, 'e', 'r', 'u', 'n', 'g', 0}
	utf8 := "Äußerung"
	z := Reader{bufr: bufio.NewReader(bytes.NewReader(latin1))}
	s, err := z.readString()
	if err != nil {
		t.Fatalf("readString: %v", err)
	}
	if s != utf8 {
		t.Fatalf("read latin-1: got %q, want %q", s, utf8)
	}

	buf := bytes.NewBuffer(make([]byte, 0, len(latin1)))
	c := Writer{w: buf}
	if _, err = c.writeString(utf8); err != nil {
		t.Fatalf("writeString: %v", err)
	}
	s = buf.String()
	if s != string(latin1) {
		t.Fatalf("write utf-8: got %q, want %q", s, string(latin1))
	}
}

// TestLatin1RoundTrip tests that metadata that is representable in Latin-1
// survives a round trip.
func TestLatin1RoundTrip(t *testing.T) {
	testCases := []struct {
		name string
		ok   bool
	}{
		{"", true},
		{"ASCII is OK", true},
		{"unless it contains a NUL\x00", false},
		{"no matter where \x00 occurs", false},
		{"\x00\x00\x00", false},
		{"Látin-1 also passes (U+00E1)", true},
		{"but LĀtin Extended-A (U+0100) does not", false},
		{"neither does 日本語", false},
		{"invalid UTF-8 also \xffails", false},
		{"\x00 as does Látin-1 with NUL", false},
	}
	for _, tc := range testCases {
		buf := new(bytes.Buffer)

		w := NewWriter(buf)
		w.Name = tc.name
		err := w.Close()
		if (err == nil) != tc.ok {
			t.Errorf("Writer.Close: name = %q, err = %v", tc.name, err)
			continue
		}
		if !tc.ok {
			continue
		}

		r, err := NewReader(buf)
		if err != nil {
			t.Errorf("NewReader: %v", err)
			continue
		}
		_, err = ioutil.ReadAll(r)
		if err != nil {
			t.Errorf("ReadAll: %v", err)
			continue
		}
		if r.Name != tc.name {
			t.Errorf("name is %q, want %q", r.Name, tc.name)
			continue
		}
		if err := r.Close(); err != nil {
			t.Errorf("Reader.Close: %v", err)
			continue
		}
	}
}

func TestWriterFlush(t *testing.T) {
	buf := new(bytes.Buffer)

	w := NewWriter(buf)
	w.Comment = "comment"
	w.Extra = []byte("extra")
	w.ModTime = time.Unix(1e8, 0)
	w.Name = "name"

	n0 := buf.Len()
	if n0 != 0 {
		t.Fatalf("buffer size = %d before writes; want 0", n0)
	}

	if err := w.Flush(); err != nil {
		t.Fatal(err)
	}

	n1 := buf.Len()
	if n1 == 0 {
		t.Fatal("no data after first flush")
	}

	w.Write([]byte("x"))

	n2 := buf.Len()
	if n1 != n2 {
		t.Fatalf("after writing a single byte, size changed from %d to %d; want no change", n1, n2)
	}

	if err := w.Flush(); err != nil {
		t.Fatal(err)
	}

	n3 := buf.Len()
	if n2 == n3 {
		t.Fatal("Flush didn't flush any data")
	}
}

// Multiple gzip files concatenated form a valid gzip file.
func TestConcat(t *testing.T) {
	var buf bytes.Buffer
	w := NewWriter(&buf)
	w.Write([]byte("hello "))
	w.Close()
	w = NewWriter(&buf)
	w.Write([]byte("world\n"))
	w.Close()

	r, err := NewReader(&buf)
	data, err := ioutil.ReadAll(r)
	if string(data) != "hello world\n" || err != nil {
		t.Fatalf("ReadAll = %q, %v, want %q, nil", data, err, "hello world")
	}
}

func TestWriterReset(t *testing.T) {
	buf := new(bytes.Buffer)
	buf2 := new(bytes.Buffer)
	z := NewWriter(buf)
	msg := []byte("hello world")
	z.Write(msg)
	z.Close()
	z.Reset(buf2)
	z.Write(msg)
	z.Close()
	if buf.String() != buf2.String() {
		t.Errorf("buf2 %q != original buf of %q", buf2.String(), buf.String())
	}
}

var testbuf []byte

func testFile(i int, t *testing.T) {
	dat, _ := ioutil.ReadFile("testdata/test.json")
	dl := len(dat)
	if len(testbuf) != i*dl {
		// Make results predictable
		testbuf = make([]byte, i*dl)
		for j := 0; j < i; j++ {
			copy(testbuf[j*dl:j*dl+dl], dat)
		}
	}

	br := bytes.NewBuffer(testbuf)
	var buf bytes.Buffer
	w, _ := NewWriterLevel(&buf, 6)
	io.Copy(w, br)
	w.Close()
	r, err := NewReader(&buf)
	if err != nil {
		t.Fatal(err.Error())
	}
	decoded, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatal(err.Error())
	}
	if !bytes.Equal(testbuf, decoded) {
		t.Errorf("decoded content does not match.")
	}
}

func TestFile1(t *testing.T)  { testFile(1, t) }
func TestFile10(t *testing.T) { testFile(10, t) }

func TestFile50(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping during short test")
	}
	testFile(50, t)
}

func TestFile200(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping during short test")
	}
	testFile(200, t)
}

func testBigGzip(i int, t *testing.T) {
	if len(testbuf) != i {
		// Make results predictable
		rand.Seed(1337)
		testbuf = make([]byte, i)
		for idx := range testbuf {
			testbuf[idx] = byte(65 + rand.Intn(32))
		}
	}

	br := bytes.NewBuffer(testbuf)
	var buf bytes.Buffer
	w, _ := NewWriterLevel(&buf, 6)
	io.Copy(w, br)
	// Test UncompressedSize()
	if len(testbuf) != int(w.UncompressedSize()) {
		t.Errorf("uncompressed size does not match. buffer:%d, UncompressedSize():%d", len(testbuf), w.UncompressedSize())
	}
	err := w.Close()
	if err != nil {
		t.Fatal(err.Error())
	}
	// Close should not affect the number
	if len(testbuf) != int(w.UncompressedSize()) {
		t.Errorf("uncompressed size does not match. buffer:%d, UncompressedSize():%d", len(testbuf), w.UncompressedSize())
	}

	r, err := NewReader(&buf)
	if err != nil {
		t.Fatal(err.Error())
	}
	decoded, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatal(err.Error())
	}
	if !bytes.Equal(testbuf, decoded) {
		t.Errorf("decoded content does not match.")
	}
}

func TestGzip1K(t *testing.T)   { testBigGzip(1000, t) }
func TestGzip100K(t *testing.T) { testBigGzip(100000, t) }
func TestGzip1M(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping during short test")
	}

	testBigGzip(1000000, t)
}
func TestGzip10M(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping during short test")
	}
	testBigGzip(10000000, t)
}

// Test if two runs produce identical results.
func TestDeterministicLM2(t *testing.T)      { testDeterm(-2, t) }
func TestDeterministicBlockLM2(t *testing.T) { testDetermBlock(-2, t) }
func TestDeterministicL0(t *testing.T)       { testDeterm(0, t) }
func TestDeterministicBlockL0(t *testing.T)  { testDetermBlock(0, t) }
func TestDeterministicL1(t *testing.T)       { testDeterm(1, t) }
func TestDeterministicBlockL1(t *testing.T)  { testDetermBlock(1, t) }
func TestDeterministicL2(t *testing.T)       { testDeterm(2, t) }
func TestDeterministicBlockL2(t *testing.T)  { testDetermBlock(2, t) }
func TestDeterministicL3(t *testing.T)       { testDeterm(3, t) }
func TestDeterministicBlockL3(t *testing.T)  { testDetermBlock(3, t) }
func TestDeterministicL4(t *testing.T)       { testDeterm(4, t) }
func TestDeterministicBlockL4(t *testing.T)  { testDetermBlock(4, t) }
func TestDeterministicL5(t *testing.T)       { testDeterm(5, t) }
func TestDeterministicBlockL5(t *testing.T)  { testDetermBlock(5, t) }
func TestDeterministicL6(t *testing.T)       { testDeterm(6, t) }
func TestDeterministicBlockL6(t *testing.T)  { testDetermBlock(6, t) }
func TestDeterministicL7(t *testing.T)       { testDeterm(7, t) }
func TestDeterministicBlockL7(t *testing.T)  { testDetermBlock(7, t) }
func TestDeterministicL8(t *testing.T)       { testDeterm(8, t) }
func TestDeterministicBlockL8(t *testing.T)  { testDetermBlock(8, t) }
func TestDeterministicL9(t *testing.T)       { testDeterm(9, t) }
func TestDeterministicBlockL9(t *testing.T)  { testDetermBlock(9, t) }

func testDeterm(i int, t *testing.T) {
	var length = defaultBlockSize*defaultBlocks + 500
	if testing.Short() {
		length = defaultBlockSize*2 + 500
	}
	rand.Seed(1337)
	t1 := make([]byte, length)
	for idx := range t1 {
		t1[idx] = byte(65 + rand.Intn(8))
	}

	br := bytes.NewBuffer(t1)
	var b1 bytes.Buffer
	w, err := NewWriterLevel(&b1, i)
	if err != nil {
		t.Fatal(err)
	}
	// Use a very small prime sized buffer.
	cbuf := make([]byte, 787)
	_, err = copyBuffer(w, br, cbuf)
	if err != nil {
		t.Fatal(err)
	}
	w.Flush()
	w.Close()

	rand.Seed(1337)
	t2 := make([]byte, length)
	for idx := range t2 {
		t2[idx] = byte(65 + rand.Intn(8))
	}

	br2 := bytes.NewBuffer(t2)
	var b2 bytes.Buffer
	w2, err := NewWriterLevel(&b2, i)
	if err != nil {
		t.Fatal(err)
	}
	// We choose a different buffer size,
	// bigger than a maximum block, and also a prime.
	cbuf = make([]byte, 81761)
	_, err = copyBuffer(w2, br2, cbuf)
	if err != nil {
		t.Fatal(err)
	}
	w2.Flush()
	w2.Close()

	b1b := b1.Bytes()
	b2b := b2.Bytes()

	if bytes.Compare(b1b, b2b) != 0 {
		t.Fatalf("Level %d did not produce deterministric result, len(a) = %d, len(b) = %d", i, len(b1b), len(b2b))
	}
}

func testDetermBlock(i int, t *testing.T) {
	var length = defaultBlockSize*defaultBlocks + 500
	if testing.Short() {
		length = defaultBlockSize*2 + 500
	}
	rand.Seed(1337)
	t1 := make([]byte, length)
	for idx := range t1 {
		t1[idx] = byte(65 + rand.Intn(8))
	}

	br := bytes.NewBuffer(t1)
	var b1 bytes.Buffer
	w, err := NewWriterLevel(&b1, i)
	if err != nil {
		t.Fatal(err)
	}
	// Use a very small prime sized buffer.
	cbuf := make([]byte, 787)
	_, err = copyBuffer(w, br, cbuf)
	if err != nil {
		t.Fatal(err)
	}
	w.Flush()
	w.Close()

	rand.Seed(1337)
	t2 := make([]byte, length)
	for idx := range t2 {
		t2[idx] = byte(65 + rand.Intn(8))
	}

	br2 := bytes.NewBuffer(t2)
	var b2 bytes.Buffer
	w2, err := NewWriterLevel(&b2, i)
	if err != nil {
		t.Fatal(err)
	}
	// We choose a different buffer size,
	// bigger than a maximum block, and also a prime.
	cbuf = make([]byte, 81761)
	_, err = copyBuffer(w2, br2, cbuf)
	if err != nil {
		t.Fatal(err)
	}
	w2.Flush()
	w2.Close()

	w1bd := w.BlockData()
	w2bd := w2.BlockData()

	if !reflect.DeepEqual(w1bd, w2bd) {
		t.Fatalf("Level %d did not produce deterministric result, a.BlockData() = %+v, b.BlockData() = %+v", i, w1bd, w2bd)
	}
}

func BenchmarkGzipL1(b *testing.B) { benchmarkGzipN(b, 1) }
func BenchmarkGzipL2(b *testing.B) { benchmarkGzipN(b, 2) }
func BenchmarkGzipL3(b *testing.B) { benchmarkGzipN(b, 3) }
func BenchmarkGzipL4(b *testing.B) { benchmarkGzipN(b, 4) }
func BenchmarkGzipL5(b *testing.B) { benchmarkGzipN(b, 5) }
func BenchmarkGzipL6(b *testing.B) { benchmarkGzipN(b, 6) }
func BenchmarkGzipL7(b *testing.B) { benchmarkGzipN(b, 7) }
func BenchmarkGzipL8(b *testing.B) { benchmarkGzipN(b, 8) }
func BenchmarkGzipL9(b *testing.B) { benchmarkGzipN(b, 9) }

func benchmarkGzipN(b *testing.B, level int) {
	dat, _ := ioutil.ReadFile("testdata/test.json")
	dat = append(dat, dat...)
	dat = append(dat, dat...)
	dat = append(dat, dat...)
	dat = append(dat, dat...)
	dat = append(dat, dat...)

	b.SetBytes(int64(len(dat)))
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		w, _ := NewWriterLevel(ioutil.Discard, level)
		w.Write(dat)
		w.Flush()
		w.Close()
	}
}

type errorWriter struct {
	mu          sync.RWMutex
	returnError bool
}

func (e *errorWriter) ErrorNow() {
	e.mu.Lock()
	e.returnError = true
	e.mu.Unlock()
}

func (e *errorWriter) Reset() {
	e.mu.Lock()
	e.returnError = false
	e.mu.Unlock()
}

func (e *errorWriter) Write(b []byte) (int, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()
	if e.returnError {
		return 0, fmt.Errorf("Intentional Error")
	}
	return len(b), nil
}

// TestErrors tests that errors are returned and that
// error state is maintained and reset by Reset.
func TestErrors(t *testing.T) {
	ew := &errorWriter{}
	w := NewWriter(ew)
	dat, _ := ioutil.ReadFile("testdata/test.json")
	n := 0
	ew.ErrorNow()
	for {
		_, err := w.Write(dat)
		if err != nil {
			break
		}
		if n > 1000 {
			t.Fatal("did not get error before 1000 iterations")
		}
		n++
	}
	if err := w.Close(); err == nil {
		t.Fatal("Writer.Close: Should have returned error")
	}
	ew.Reset()
	w.Reset(ew)
	_, err := w.Write(dat)
	if err != nil {
		t.Fatal("Writer after Reset, unexpected error:", err)
	}
	ew.ErrorNow()
	if err = w.Flush(); err == nil {
		t.Fatal("Writer.Flush: Should have returned error")
	}
	if err = w.Close(); err == nil {
		t.Fatal("Writer.Close: Should have returned error")
	}
	// Test Sync only
	w.Reset(ew)
	if err = w.Flush(); err == nil {
		t.Fatal("Writer.Flush: Should have returned error")
	}
	if err = w.Close(); err == nil {
		t.Fatal("Writer.Close: Should have returned error")
	}
	// Test Close only
	w.Reset(ew)
	if err = w.Close(); err == nil {
		t.Fatal("Writer.Close: Should have returned error")
	}

}

// A writer that fails after N bytes.
type errorWriter2 struct {
	N int
}

func (e *errorWriter2) Write(b []byte) (int, error) {
	e.N -= len(b)
	if e.N <= 0 {
		return 0, io.ErrClosedPipe
	}
	return len(b), nil
}

// Test if errors from the underlying writer is passed upwards.
func TestWriteError(t *testing.T) {
	n := defaultBlockSize + 1024
	if !testing.Short() {
		n *= 4
	}
	// Make it incompressible...
	in := make([]byte, n+1<<10)
	io.ReadFull(rand.New(rand.NewSource(0xabad1dea)), in)

	// We create our own buffer to control number of writes.
	copyBuf := make([]byte, 128)
	for l := 0; l < 10; l++ {
		t.Run("level-"+strconv.Itoa(l), func(t *testing.T) {
			for fail := 1; fail < n; fail *= 10 {
				// Fail after 'fail' writes
				ew := &errorWriter2{N: fail}
				w, err := NewWriterLevel(ew, l)
				if err != nil {
					t.Fatalf("NewWriter: level %d: %v", l, err)
				}
				// Set concurrency low enough that errors should propagate.
				w.SetConcurrency(128<<10, 4)
				_, err = copyBuffer(w, bytes.NewBuffer(in), copyBuf)
				if err == nil {
					t.Errorf("Level %d: Expected an error, writer was %#v", l, ew)
				}
				n2, err := w.Write([]byte{1, 2, 2, 3, 4, 5})
				if n2 != 0 {
					t.Error("Level", l, "Expected 0 length write, got", n2)
				}
				if err == nil {
					t.Error("Level", l, "Expected an error")
				}
				err = w.Flush()
				if err == nil {
					t.Error("Level", l, "Expected an error on flush")
				}
				err = w.Close()
				if err == nil {
					t.Error("Level", l, "Expected an error on close")
				}

				w.Reset(ioutil.Discard)
				n2, err = w.Write([]byte{1, 2, 3, 4, 5, 6})
				if err != nil {
					t.Error("Level", l, "Got unexpected error after reset:", err)
				}
				if n2 == 0 {
					t.Error("Level", l, "Got 0 length write, expected > 0")
				}
				if testing.Short() {
					return
				}
			}
		})
	}
}

// copyBuffer is a copy of io.CopyBuffer, since we want to support older go versions.
// This is modified to never use io.WriterTo or io.ReaderFrom interfaces.
func copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
	if buf == nil {
		buf = make([]byte, 32*1024)
	}
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = io.ErrShortWrite
				break
			}
		}
		if er == io.EOF {
			break
		}
		if er != nil {
			err = er
			break
		}
	}
	return written, err
}