Debian Codegraph @

compress/snappy/xerial_test.go

raw
1 package snappy
2
3 import (
4 "bytes"
5 "crypto/rand"
6 "io"
7 "testing"
8
9 "github.com/klauspost/compress/snappy"
10 goxerialsnappy "github.com/segmentio/kafka-go/compress/snappy/go-xerial-snappy"
11 )
12
13 // Wrap an io.Reader or io.Writer to disable all copy optimizations like
14 // io.WriterTo or io.ReaderFrom.
15 // We use this to ensure writes are chunked by io.Copy's internal buffer
16 // in the tests.
17 type simpleReader struct{ io.Reader }
18 type simpleWriter struct{ io.Writer }
19
20 func TestXerialReaderSnappy(t *testing.T) {
21 rawData := new(bytes.Buffer)
22 rawData.Grow(1024 * 1024)
23 io.CopyN(rawData, rand.Reader, 1024*1024)
24
25 compressedRawData := bytes.NewReader(snappy.Encode(nil, rawData.Bytes()))
26
27 decompressedData := new(bytes.Buffer)
28 io.Copy(decompressedData,
29 &xerialReader{reader: compressedRawData, decode: snappy.Decode})
30
31 b0 := rawData.Bytes()
32 b1 := decompressedData.Bytes()
33
34 if !bytes.Equal(b0, b1) {
35 t.Error("data mismatch")
36 }
37 }
38
39 func TestXerialReaderWriter(t *testing.T) {
40 rawData := new(bytes.Buffer)
41 rawData.Grow(1024 * 1024)
42 io.CopyN(rawData, rand.Reader, 1024*1024)
43
44 framedData := new(bytes.Buffer)
45 framedData.Grow(rawData.Len() + 1024)
46 w := simpleWriter{&xerialWriter{writer: framedData}}
47 r := simpleReader{bytes.NewReader(rawData.Bytes())}
48 io.Copy(w, r)
49 w.Writer.(*xerialWriter).Flush()
50
51 unframedData := new(bytes.Buffer)
52 unframedData.Grow(rawData.Len())
53 io.Copy(unframedData, &xerialReader{reader: framedData})
54
55 b0 := rawData.Bytes()
56 b1 := unframedData.Bytes()
57
58 if !bytes.Equal(b0, b1) {
59 t.Error("data mismatch")
60 }
61 }
62
63 func TestXerialFramedCompression(t *testing.T) {
64 rawData := new(bytes.Buffer)
65 rawData.Grow(1024 * 1024)
66 io.CopyN(rawData, rand.Reader, 1024*1024)
67
68 framedAndCompressedData := new(bytes.Buffer)
69 framedAndCompressedData.Grow(rawData.Len())
70 w := simpleWriter{&xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}}
71 r := simpleReader{bytes.NewReader(rawData.Bytes())}
72 io.Copy(w, r)
73 w.Writer.(*xerialWriter).Flush()
74
75 unframedAndDecompressedData := new(bytes.Buffer)
76 unframedAndDecompressedData.Grow(rawData.Len())
77 io.Copy(unframedAndDecompressedData,
78 simpleReader{&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode}})
79
80 b0 := rawData.Bytes()
81 b1 := unframedAndDecompressedData.Bytes()
82
83 if !bytes.Equal(b0, b1) {
84 t.Error("data mismatch")
85 }
86 }
87
88 func TestXerialFramedCompressionOptimized(t *testing.T) {
89 rawData := new(bytes.Buffer)
90 rawData.Grow(1024 * 1024)
91 io.CopyN(rawData, rand.Reader, 1024*1024)
92
93 framedAndCompressedData := new(bytes.Buffer)
94 framedAndCompressedData.Grow(rawData.Len())
95 w := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}
96 r := simpleReader{bytes.NewReader(rawData.Bytes())}
97 io.Copy(w, r)
98 w.Flush()
99
100 unframedAndDecompressedData := new(bytes.Buffer)
101 unframedAndDecompressedData.Grow(rawData.Len())
102 io.Copy(unframedAndDecompressedData,
103 &xerialReader{reader: framedAndCompressedData, decode: snappy.Decode})
104
105 b0 := rawData.Bytes()
106 b1 := unframedAndDecompressedData.Bytes()
107
108 if !bytes.Equal(b0, b1) {
109 t.Error("data mismatch")
110 }
111 }
112
113 func TestXerialReaderAgainstGoXerialSnappy(t *testing.T) {
114 rawData := new(bytes.Buffer)
115 rawData.Grow(1024 * 1024)
116 io.CopyN(rawData, rand.Reader, 1024*1024)
117 rawBytes := rawData.Bytes()
118
119 framedAndCompressedData := []byte{}
120 const chunkSize = 999
121 for i := 0; i < len(rawBytes); i += chunkSize {
122 j := i + chunkSize
123 if j > len(rawBytes) {
124 j = len(rawBytes)
125 }
126 framedAndCompressedData = goxerialsnappy.EncodeStream(framedAndCompressedData, rawBytes[i:j])
127 }
128
129 unframedAndDecompressedData := new(bytes.Buffer)
130 unframedAndDecompressedData.Grow(rawData.Len())
131 io.Copy(unframedAndDecompressedData,
132 &xerialReader{reader: bytes.NewReader(framedAndCompressedData), decode: snappy.Decode})
133
134 b0 := rawBytes
135 b1 := unframedAndDecompressedData.Bytes()
136
137 if !bytes.Equal(b0, b1) {
138 t.Error("data mismatch")
139 }
140 }
141
142 func TestXerialWriterAgainstGoXerialSnappy(t *testing.T) {
143 rawData := new(bytes.Buffer)
144 rawData.Grow(1024 * 1024)
145 io.CopyN(rawData, rand.Reader, 1024*1024)
146
147 framedAndCompressedData := new(bytes.Buffer)
148 framedAndCompressedData.Grow(rawData.Len())
149 w := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}
150 r := simpleReader{bytes.NewReader(rawData.Bytes())}
151 io.Copy(w, r)
152 w.Flush()
153
154 unframedAndDecompressedData, err := goxerialsnappy.Decode(framedAndCompressedData.Bytes())
155 if err != nil {
156 t.Error(err)
157 }
158
159 b0 := rawData.Bytes()
160 b1 := unframedAndDecompressedData
161
162 if !bytes.Equal(b0, b1) {
163 t.Error("data mismatch")
164 }
165 }