| 1 |
package snappy |
| 2 |
|
| 3 |
import ( |
| 4 |
"bytes" |
| 5 |
"crypto/rand" |
| 6 |
"io" |
| 7 |
"testing" |
| 8 |
|
| 9 |
"github.com/klauspost/compress/snappy" |
| 10 |
goxerialsnappy "github.com/segmentio/kafka-go/compress/snappy/go-xerial-snappy" |
| 11 |
) |
| 12 |
|
| 13 |
// Wrap an io.Reader or io.Writer to disable all copy optimizations like |
| 14 |
// io.WriterTo or io.ReaderFrom. |
| 15 |
// We use this to ensure writes are chunked by io.Copy's internal buffer |
| 16 |
// in the tests. |
| 17 |
type simpleReader struct{ io.Reader } |
| 18 |
type simpleWriter struct{ io.Writer } |
| 19 |
|
| 20 |
func TestXerialReaderSnappy(t *testing.T) { |
| 21 |
rawData := new(bytes.Buffer) |
| 22 |
rawData.Grow(1024 * 1024) |
| 23 |
io.CopyN(rawData, rand.Reader, 1024*1024) |
| 24 |
|
| 25 |
compressedRawData := bytes.NewReader(snappy.Encode(nil, rawData.Bytes())) |
| 26 |
|
| 27 |
decompressedData := new(bytes.Buffer) |
| 28 |
io.Copy(decompressedData, |
| 29 |
&xerialReader{reader: compressedRawData, decode: snappy.Decode}) |
| 30 |
|
| 31 |
b0 := rawData.Bytes() |
| 32 |
b1 := decompressedData.Bytes() |
| 33 |
|
| 34 |
if !bytes.Equal(b0, b1) { |
| 35 |
t.Error("data mismatch") |
| 36 |
} |
| 37 |
} |
| 38 |
|
| 39 |
func TestXerialReaderWriter(t *testing.T) { |
| 40 |
rawData := new(bytes.Buffer) |
| 41 |
rawData.Grow(1024 * 1024) |
| 42 |
io.CopyN(rawData, rand.Reader, 1024*1024) |
| 43 |
|
| 44 |
framedData := new(bytes.Buffer) |
| 45 |
framedData.Grow(rawData.Len() + 1024) |
| 46 |
w := simpleWriter{&xerialWriter{writer: framedData}} |
| 47 |
r := simpleReader{bytes.NewReader(rawData.Bytes())} |
| 48 |
io.Copy(w, r) |
| 49 |
w.Writer.(*xerialWriter).Flush() |
| 50 |
|
| 51 |
unframedData := new(bytes.Buffer) |
| 52 |
unframedData.Grow(rawData.Len()) |
| 53 |
io.Copy(unframedData, &xerialReader{reader: framedData}) |
| 54 |
|
| 55 |
b0 := rawData.Bytes() |
| 56 |
b1 := unframedData.Bytes() |
| 57 |
|
| 58 |
if !bytes.Equal(b0, b1) { |
| 59 |
t.Error("data mismatch") |
| 60 |
} |
| 61 |
} |
| 62 |
|
| 63 |
func TestXerialFramedCompression(t *testing.T) { |
| 64 |
rawData := new(bytes.Buffer) |
| 65 |
rawData.Grow(1024 * 1024) |
| 66 |
io.CopyN(rawData, rand.Reader, 1024*1024) |
| 67 |
|
| 68 |
framedAndCompressedData := new(bytes.Buffer) |
| 69 |
framedAndCompressedData.Grow(rawData.Len()) |
| 70 |
w := simpleWriter{&xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}} |
| 71 |
r := simpleReader{bytes.NewReader(rawData.Bytes())} |
| 72 |
io.Copy(w, r) |
| 73 |
w.Writer.(*xerialWriter).Flush() |
| 74 |
|
| 75 |
unframedAndDecompressedData := new(bytes.Buffer) |
| 76 |
unframedAndDecompressedData.Grow(rawData.Len()) |
| 77 |
io.Copy(unframedAndDecompressedData, |
| 78 |
simpleReader{&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode}}) |
| 79 |
|
| 80 |
b0 := rawData.Bytes() |
| 81 |
b1 := unframedAndDecompressedData.Bytes() |
| 82 |
|
| 83 |
if !bytes.Equal(b0, b1) { |
| 84 |
t.Error("data mismatch") |
| 85 |
} |
| 86 |
} |
| 87 |
|
| 88 |
func TestXerialFramedCompressionOptimized(t *testing.T) { |
| 89 |
rawData := new(bytes.Buffer) |
| 90 |
rawData.Grow(1024 * 1024) |
| 91 |
io.CopyN(rawData, rand.Reader, 1024*1024) |
| 92 |
|
| 93 |
framedAndCompressedData := new(bytes.Buffer) |
| 94 |
framedAndCompressedData.Grow(rawData.Len()) |
| 95 |
w := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode} |
| 96 |
r := simpleReader{bytes.NewReader(rawData.Bytes())} |
| 97 |
io.Copy(w, r) |
| 98 |
w.Flush() |
| 99 |
|
| 100 |
unframedAndDecompressedData := new(bytes.Buffer) |
| 101 |
unframedAndDecompressedData.Grow(rawData.Len()) |
| 102 |
io.Copy(unframedAndDecompressedData, |
| 103 |
&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode}) |
| 104 |
|
| 105 |
b0 := rawData.Bytes() |
| 106 |
b1 := unframedAndDecompressedData.Bytes() |
| 107 |
|
| 108 |
if !bytes.Equal(b0, b1) { |
| 109 |
t.Error("data mismatch") |
| 110 |
} |
| 111 |
} |
| 112 |
|
| 113 |
func TestXerialReaderAgainstGoXerialSnappy(t *testing.T) { |
| 114 |
rawData := new(bytes.Buffer) |
| 115 |
rawData.Grow(1024 * 1024) |
| 116 |
io.CopyN(rawData, rand.Reader, 1024*1024) |
| 117 |
rawBytes := rawData.Bytes() |
| 118 |
|
| 119 |
framedAndCompressedData := []byte{} |
| 120 |
const chunkSize = 999 |
| 121 |
for i := 0; i < len(rawBytes); i += chunkSize { |
| 122 |
j := i + chunkSize |
| 123 |
if j > len(rawBytes) { |
| 124 |
j = len(rawBytes) |
| 125 |
} |
| 126 |
framedAndCompressedData = goxerialsnappy.EncodeStream(framedAndCompressedData, rawBytes[i:j]) |
| 127 |
} |
| 128 |
|
| 129 |
unframedAndDecompressedData := new(bytes.Buffer) |
| 130 |
unframedAndDecompressedData.Grow(rawData.Len()) |
| 131 |
io.Copy(unframedAndDecompressedData, |
| 132 |
&xerialReader{reader: bytes.NewReader(framedAndCompressedData), decode: snappy.Decode}) |
| 133 |
|
| 134 |
b0 := rawBytes |
| 135 |
b1 := unframedAndDecompressedData.Bytes() |
| 136 |
|
| 137 |
if !bytes.Equal(b0, b1) { |
| 138 |
t.Error("data mismatch") |
| 139 |
} |
| 140 |
} |
| 141 |
|
| 142 |
func TestXerialWriterAgainstGoXerialSnappy(t *testing.T) { |
| 143 |
rawData := new(bytes.Buffer) |
| 144 |
rawData.Grow(1024 * 1024) |
| 145 |
io.CopyN(rawData, rand.Reader, 1024*1024) |
| 146 |
|
| 147 |
framedAndCompressedData := new(bytes.Buffer) |
| 148 |
framedAndCompressedData.Grow(rawData.Len()) |
| 149 |
w := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode} |
| 150 |
r := simpleReader{bytes.NewReader(rawData.Bytes())} |
| 151 |
io.Copy(w, r) |
| 152 |
w.Flush() |
| 153 |
|
| 154 |
unframedAndDecompressedData, err := goxerialsnappy.Decode(framedAndCompressedData.Bytes()) |
| 155 |
if err != nil { |
| 156 |
t.Error(err) |
| 157 |
} |
| 158 |
|
| 159 |
b0 := rawData.Bytes() |
| 160 |
b1 := unframedAndDecompressedData |
| 161 |
|
| 162 |
if !bytes.Equal(b0, b1) { |
| 163 |
t.Error("data mismatch") |
| 164 |
} |
| 165 |
} |