| 1 |
// Copyright 2009 The Go Authors. All rights reserved. |
| 2 |
// Use of this source code is governed by a BSD-style |
| 3 |
// license that can be found in the Go LICENSE file. |
| 4 |
|
| 5 |
// Changes copyright ©2012 The bíogo Authors. All rights reserved. |
| 6 |
// Use of this source code is governed by a BSD-style |
| 7 |
// license that can be found in the LICENSE file. |
| 8 |
|
| 9 |
package bgzf_test |
| 10 |
|
| 11 |
import ( |
| 12 |
"bytes" |
| 13 |
"compress/gzip" |
| 14 |
"errors" |
| 15 |
"flag" |
| 16 |
"fmt" |
| 17 |
"io" |
| 18 |
"os" |
| 19 |
"path/filepath" |
| 20 |
"runtime" |
| 21 |
"strings" |
| 22 |
"sync" |
| 23 |
"testing" |
| 24 |
"time" |
| 25 |
|
| 26 |
. "github.com/biogo/hts/bgzf" |
| 27 |
"github.com/biogo/hts/bgzf/cache" |
| 28 |
) |
| 29 |
|
| 30 |
var ( |
| 31 |
conc = flag.Int("conc", 1, "sets the level of concurrency for compression") |
| 32 |
file = flag.String("bench.file", "", "bgzf file to read for benchmarking decompression") |
| 33 |
) |
| 34 |
|
| 35 |
type countWriter struct { |
| 36 |
bytes int64 |
| 37 |
w io.Writer |
| 38 |
} |
| 39 |
|
| 40 |
func (cw *countWriter) Write(p []byte) (n int, err error) { |
| 41 |
n, err = cw.w.Write(p) |
| 42 |
cw.bytes += int64(n) |
| 43 |
return |
| 44 |
} |
| 45 |
|
| 46 |
// TestEmpty tests that an empty payload still forms a valid GZIP stream. |
| 47 |
func TestEmpty(t *testing.T) { |
| 48 |
buf := new(bytes.Buffer) |
| 49 |
|
| 50 |
if err := NewWriter(buf, *conc).Close(); err != nil { |
| 51 |
t.Fatalf("Writer.Close: %v", err) |
| 52 |
} |
| 53 |
|
| 54 |
r, err := NewReader(buf, *conc) |
| 55 |
if err != nil { |
| 56 |
t.Fatalf("NewReader: %v", err) |
| 57 |
} |
| 58 |
b, err := io.ReadAll(r) |
| 59 |
if err != nil { |
| 60 |
t.Fatalf("ReadAll: %v", err) |
| 61 |
} |
| 62 |
if len(b) != 0 { |
| 63 |
t.Fatalf("got %d bytes, want 0", len(b)) |
| 64 |
} |
| 65 |
if err := r.Close(); err != nil { |
| 66 |
t.Fatalf("Reader.Close: %v", err) |
| 67 |
} |
| 68 |
} |
| 69 |
|
| 70 |
type crippledReaderAt struct { |
| 71 |
r *bytes.Reader |
| 72 |
} |
| 73 |
|
| 74 |
func (r crippledReaderAt) ReadAt(b []byte, off int64) (int, error) { |
| 75 |
return r.r.ReadAt(b, off) |
| 76 |
} |
| 77 |
|
| 78 |
// TestEOF tests HasEOF can find the EOF magic block. |
| 79 |
func TestEOF(t *testing.T) { |
| 80 |
f, err := os.Create(filepath.Join(t.TempDir(), "data")) |
| 81 |
if err != nil { |
| 82 |
t.Fatalf("Create temp file: %v", err) |
| 83 |
} |
| 84 |
fname := f.Name() |
| 85 |
|
| 86 |
if err := NewWriter(f, *conc).Close(); err != nil { |
| 87 |
t.Fatalf("Writer.Close: %v", err) |
| 88 |
} |
| 89 |
|
| 90 |
if err := f.Close(); err != nil { |
| 91 |
t.Fatalf("f.Close: %v", err) |
| 92 |
} |
| 93 |
|
| 94 |
f, err = os.Open(fname) |
| 95 |
if err != nil { |
| 96 |
t.Fatalf("Open temp file: %v", err) |
| 97 |
} |
| 98 |
ok, err := HasEOF(f) |
| 99 |
if err != nil { |
| 100 |
t.Errorf("HasEOF: %v", err) |
| 101 |
} |
| 102 |
if !ok { |
| 103 |
t.Error("Expected EOF in os.File: not found.") |
| 104 |
} |
| 105 |
if err := f.Close(); err != nil { |
| 106 |
t.Fatalf("f.Close: %v", err) |
| 107 |
} |
| 108 |
|
| 109 |
if runtime.GOOS != "windows" { |
| 110 |
// NOTE: on Windows, os.Open is not allowed for directories. |
| •111 |
tempdir := t.TempDir() |
| 112 |
f, err = os.Open(tempdir) |
| 113 |
if err != nil { |
| 114 |
t.Fatalf("Open temp dir: %v", err) |
| 115 |
} |
| 116 |
ok, err = HasEOF(f) |
| •117 |
if want := "read " + tempdir + ": is a directory"; err.Error() != want { |
| 118 |
t.Errorf("Expected error:%s got:%v", want, err) |
| 119 |
} |
| 120 |
if ok { |
| 121 |
t.Error("Unexpected EOF in os.File IsDir: found.") |
| 122 |
} |
| 123 |
if err := f.Close(); err != nil { |
| 124 |
t.Fatalf("f.Close: %v", err) |
| 125 |
} |
| 126 |
} |
| 127 |
|
| 128 |
// {bytes,strings}.Reader cases |
| 129 |
var buf bytes.Buffer |
| 130 |
if err := NewWriter(&buf, *conc).Close(); err != nil { |
| 131 |
t.Fatalf("Writer.Close: %v", err) |
| 132 |
} |
| 133 |
|
| 134 |
ok, err = HasEOF(bytes.NewReader(buf.Bytes())) |
| 135 |
if err != nil { |
| 136 |
t.Errorf("HasEOF: %v", err) |
| 137 |
} |
| 138 |
if !ok { |
| 139 |
t.Error("Expected EOF in []byte: not found.") |
| 140 |
} |
| 141 |
|
| 142 |
ok, err = HasEOF(strings.NewReader(buf.String())) |
| 143 |
if err != nil { |
| 144 |
t.Errorf("HasEOF: %v", err) |
| 145 |
} |
| 146 |
if !ok { |
| 147 |
t.Error("Expected EOF in string: not found.") |
| 148 |
} |
| 149 |
|
| 150 |
ok, err = HasEOF(crippledReaderAt{bytes.NewReader(buf.Bytes())}) |
| 151 |
if err != ErrNoEnd { |
| 152 |
t.Errorf("Expected error:%s got:%v", ErrNoEnd, err) |
| 153 |
} |
| 154 |
if ok { |
| 155 |
t.Error("Unexpected EOF in crippled ReaderAt: found.") |
| 156 |
} |
| 157 |
} |
| 158 |
|
| 159 |
// TestRoundTrip tests that bgzipping and then bgunzipping is the identity |
| 160 |
// function. |
| 161 |
func TestRoundTrip(t *testing.T) { |
| 162 |
for _, reader := range []struct { |
| 163 |
name string |
| 164 |
readAll func(*Reader) ([]byte, error) |
| 165 |
}{ |
| 166 |
{"io.Reader", readAllWrapper}, |
| 167 |
{"io.ByteReader", readAllByByte}, |
| 168 |
} { |
| 169 |
buf := new(bytes.Buffer) |
| 170 |
|
| 171 |
w := NewWriter(buf, *conc) |
| 172 |
w.Comment = "comment" |
| 173 |
w.Extra = []byte("extra") |
| 174 |
w.ModTime = time.Unix(1e8, 0) |
| 175 |
w.Name = "name" |
| 176 |
if _, err := w.Write([]byte("payload")); err != nil { |
| 177 |
t.Fatalf("Write: %v", err) |
| 178 |
} |
| 179 |
if err := w.Close(); err != nil { |
| 180 |
t.Fatalf("Writer.Close: %v", err) |
| 181 |
} |
| 182 |
// FIXME(kortschak) The magic block is written on close, |
| 183 |
// so we need to discount that until we have the capacity |
| 184 |
// to see every header again. |
| 185 |
wbl := buf.Len() - len(MagicBlock) |
| 186 |
|
| 187 |
r, err := NewReader(buf, *conc) |
| 188 |
if err != nil { |
| 189 |
t.Fatalf("NewReader: %v", err) |
| 190 |
} |
| 191 |
|
| 192 |
if bl := ExpectedMemberSize(r.Header); bl != wbl { |
| 193 |
t.Errorf("expectedMemberSize is %d, want %d", bl, wbl) |
| 194 |
} |
| 195 |
blEnc := string([]byte{byte(wbl - 1), byte((wbl - 1) >> 8)}) |
| 196 |
if string(r.Extra) != "BC\x02\x00"+blEnc+"extra" { |
| 197 |
t.Errorf("extra is %q, want %q", r.Extra, "BC\x02\x00"+blEnc+"extra") |
| 198 |
} |
| 199 |
b, err := reader.readAll(r) |
| 200 |
if err != nil { |
| 201 |
t.Fatalf("%s readAll: %v", reader.name, err) |
| 202 |
} |
| 203 |
if string(b) != "payload" { |
| 204 |
t.Fatalf("%s payload is %q, want %q", reader.name, string(b), "payload") |
| 205 |
} |
| 206 |
if r.Comment != "comment" { |
| 207 |
t.Errorf("comment is %q, want %q", r.Comment, "comment") |
| 208 |
} |
| 209 |
if bl := ExpectedMemberSize(r.Header); bl != len(MagicBlock) { |
| 210 |
t.Errorf("expectedMemberSize is %d, want %d", bl, len(MagicBlock)) |
| 211 |
} |
| 212 |
if string(r.Extra) != "BC\x02\x00\x1b\x00" { |
| 213 |
t.Errorf("extra is %q, want %q", r.Extra, "BC\x02\x00\x1b\x00") |
| 214 |
} |
| 215 |
if r.ModTime.Unix() != 1e8 { |
| 216 |
t.Errorf("mtime is %d, want %d", r.ModTime.Unix(), uint32(1e8)) |
| 217 |
} |
| 218 |
if r.Name != "name" { |
| 219 |
t.Errorf("name is %q, want %q", r.Name, "name") |
| 220 |
} |
| 221 |
if r.OS != 0xff { |
| 222 |
t.Errorf("os is %x, want %x", r.OS, 0xff) |
| 223 |
} |
| 224 |
if err := r.Close(); err != nil { |
| 225 |
t.Errorf("Reader.Close: %v", err) |
| 226 |
} |
| 227 |
} |
| 228 |
} |
| 229 |
|
| 230 |
func readAllWrapper(r *Reader) ([]byte, error) { |
| 231 |
return io.ReadAll(r) |
| 232 |
} |
| 233 |
|
| 234 |
func readAllByByte(r *Reader) ([]byte, error) { |
| 235 |
var ( |
| 236 |
buf []byte |
| 237 |
err error |
| 238 |
b byte |
| 239 |
) |
| 240 |
for { |
| 241 |
b, err = r.ReadByte() |
| 242 |
if err != nil { |
| 243 |
break |
| 244 |
} |
| 245 |
buf = append(buf, b) |
| 246 |
} |
| 247 |
if err == io.EOF { |
| 248 |
err = nil |
| 249 |
} |
| 250 |
return buf, err |
| 251 |
} |
| 252 |
|
| 253 |
// TestRoundTripMulti tests that bgzipping and then bgunzipping is the identity |
| 254 |
// function for a multiple member bgzf. |
| 255 |
func TestRoundTripMulti(t *testing.T) { |
| 256 |
for _, reader := range []struct { |
| 257 |
name string |
| 258 |
read func(*Reader, []byte) (int, error) |
| 259 |
}{ |
| 260 |
{"io.Reader", readWrapper}, |
| 261 |
{"io.ByteReader", readByByte}, |
| 262 |
} { |
| 263 |
var wbl [2]int |
| 264 |
buf := new(bytes.Buffer) |
| 265 |
|
| 266 |
w := NewWriter(buf, *conc) |
| 267 |
w.Comment = "comment" |
| 268 |
w.Extra = []byte("extra") |
| 269 |
w.ModTime = time.Unix(1e8, 0) |
| 270 |
w.Name = "name" |
| 271 |
if _, err := w.Write([]byte("payload1")); err != nil { |
| 272 |
t.Fatalf("Write: %v", err) |
| 273 |
} |
| 274 |
if err := w.Flush(); err != nil { |
| 275 |
t.Fatalf("Flush: %v", err) |
| 276 |
} |
| 277 |
if err := w.Wait(); err != nil { |
| 278 |
t.Fatalf("Wait: %v", err) |
| 279 |
} |
| 280 |
wbl[0] = buf.Len() |
| 281 |
if _, err := w.Write([]byte("payloadTwo")); err != nil { |
| 282 |
t.Fatalf("Write: %v", err) |
| 283 |
} |
| 284 |
if err := w.Close(); err != nil { |
| 285 |
t.Fatalf("Writer.Close: %v", err) |
| 286 |
} |
| 287 |
wbl[1] = buf.Len() - wbl[0] - len(MagicBlock) |
| 288 |
|
| 289 |
var ( |
| 290 |
b []byte |
| 291 |
bl, n int |
| 292 |
err error |
| 293 |
) |
| 294 |
r, err := NewReader(buf, *conc) |
| 295 |
if err != nil { |
| 296 |
t.Fatalf("NewReader: %v", err) |
| 297 |
} |
| 298 |
|
| 299 |
if r.Comment != "comment" { |
| 300 |
t.Errorf("comment is %q, want %q", r.Comment, "comment") |
| 301 |
} |
| 302 |
blEnc := string([]byte{byte(wbl[0] - 1), byte((wbl[0] - 1) >> 8)}) |
| 303 |
if string(r.Extra) != "BC\x02\x00"+blEnc+"extra" { |
| 304 |
t.Errorf("extra is %q, want %q", r.Extra, "BC\x02\x00"+blEnc+"extra") |
| 305 |
} |
| 306 |
if r.ModTime.Unix() != 1e8 { |
| 307 |
t.Errorf("mtime is %d, want %d", r.ModTime.Unix(), uint32(1e8)) |
| 308 |
} |
| 309 |
if r.Name != "name" { |
| 310 |
t.Errorf("name is %q, want %q", r.Name, "name") |
| 311 |
} |
| 312 |
|
| 313 |
bl = ExpectedMemberSize(r.Header) |
| 314 |
if bl != wbl[0] { |
| 315 |
t.Errorf("expectedMemberSize is %d, want %d", bl, wbl[0]) |
| 316 |
} |
| 317 |
b = make([]byte, len("payload1payloadTwo")) |
| 318 |
n, err = reader.read(r, b) |
| 319 |
if string(b[:n]) != "payload1payloadTwo" { |
| 320 |
t.Errorf("%s payload is %q, want %q", reader.name, string(b[:n]), "payload1payloadTwo") |
| 321 |
} |
| 322 |
if err != nil { |
| 323 |
t.Errorf("%s read: %v", reader.name, err) |
| 324 |
} |
| 325 |
|
| 326 |
bl = ExpectedMemberSize(r.Header) |
| 327 |
if bl != wbl[1] { |
| 328 |
t.Errorf("expectedMemberSize is %d, want %d", bl, wbl[1]) |
| 329 |
} |
| 330 |
b = make([]byte, 1) |
| 331 |
n, err = reader.read(r, b) |
| 332 |
if string(b[:n]) != "" { |
| 333 |
t.Errorf("%s payload is %q, want %q", reader.name, string(b[:n]), "") |
| 334 |
} |
| 335 |
if err != io.EOF { |
| 336 |
t.Errorf("%s read: %v", reader.name, err) |
| 337 |
} |
| 338 |
r.Close() |
| 339 |
} |
| 340 |
} |
| 341 |
|
| 342 |
func readWrapper(r *Reader, buf []byte) (int, error) { |
| 343 |
return r.Read(buf) |
| 344 |
} |
| 345 |
|
| 346 |
func readByByte(r *Reader, buf []byte) (n int, err error) { |
| 347 |
for range buf { |
| 348 |
buf[n], err = r.ReadByte() |
| 349 |
if err != nil { |
| 350 |
break |
| 351 |
} |
| 352 |
n++ |
| 353 |
} |
| 354 |
return n, err |
| 355 |
} |
| 356 |
|
| 357 |
// See https://github.com/biogo/hts/issues/57 |
| 358 |
func TestHeaderIssue57(t *testing.T) { |
| 359 |
var stamp time.Time |
| 360 |
|
| 361 |
var buf bytes.Buffer |
| 362 |
bg := NewWriter(&buf, *conc) |
| 363 |
bg.ModTime = stamp |
| 364 |
bg.OS = 0xff |
| 365 |
err := bg.Close() |
| 366 |
if err != nil { |
| 367 |
t.Fatal("error closing Writer") |
| 368 |
} |
| 369 |
got := buf.Bytes()[:16] |
| 370 |
want := []byte(MagicBlock[:16]) |
| 371 |
if !bytes.Equal(got, want) { |
| 372 |
t.Fatalf("unexpected header:\ngot: %0#2v\nwant:%0#2v", got, want) |
| 373 |
} |
| 374 |
} |
| 375 |
|
| 376 |
// TestRoundTripMultiSeek tests that bgzipping and then bgunzipping is the identity |
| 377 |
// function for a multiple member bgzf with an underlying Seeker. |
| 378 |
func TestRoundTripMultiSeek(t *testing.T) { |
| 379 |
f, err := os.Create(filepath.Join(t.TempDir(), "data")) |
| 380 |
if err != nil { |
| 381 |
t.Fatalf("Create temp file: %v", err) |
| 382 |
} |
| 383 |
fname := f.Name() |
| 384 |
|
| 385 |
var wbl [2]int |
| 386 |
cw := &countWriter{w: f} |
| 387 |
w := NewWriter(cw, *conc) |
| 388 |
w.Comment = "comment" |
| 389 |
w.Extra = []byte("extra") |
| 390 |
w.ModTime = time.Unix(1e8, 0) |
| 391 |
w.Name = "name" |
| 392 |
if _, err := w.Write([]byte("payload1")); err != nil { |
| 393 |
t.Fatalf("Write: %v", err) |
| 394 |
} |
| 395 |
if err := w.Flush(); err != nil { |
| 396 |
t.Fatalf("Flush: %v", err) |
| 397 |
} |
| 398 |
if err := w.Wait(); err != nil { |
| 399 |
t.Fatalf("Wait: %v", err) |
| 400 |
} |
| 401 |
offset := cw.bytes |
| 402 |
wbl[0] = int(offset) |
| 403 |
if _, err := w.Write([]byte("payloadTwo")); err != nil { |
| 404 |
t.Fatalf("Write: %v", err) |
| 405 |
} |
| 406 |
if err := w.Close(); err != nil { |
| 407 |
t.Fatalf("Writer.Close: %v", err) |
| 408 |
} |
| 409 |
if err := f.Close(); err != nil { |
| 410 |
t.Fatalf("os.File.Close: %v", err) |
| 411 |
} |
| 412 |
wbl[1] = int(cw.bytes-offset) - len(MagicBlock) |
| 413 |
|
| 414 |
var ( |
| 415 |
b []byte |
| 416 |
bl, n int |
| 417 |
) |
| 418 |
|
| 419 |
f, err = os.Open(fname) |
| 420 |
if err != nil { |
| 421 |
t.Fatalf("Reopen temp file: %v", err) |
| 422 |
} |
| 423 |
r, err := NewReader(f, *conc) |
| 424 |
if err != nil { |
| 425 |
t.Fatalf("NewReader: %v", err) |
| 426 |
} |
| 427 |
|
| 428 |
// Insert a HasEOF to ensure it does not corrupt subsequent reads. |
| 429 |
HasEOF(f) |
| 430 |
|
| 431 |
if r.Comment != "comment" { |
| 432 |
t.Errorf("comment is %q, want %q", r.Comment, "comment") |
| 433 |
} |
| 434 |
blEnc := string([]byte{byte(wbl[0] - 1), byte((wbl[0] - 1) >> 8)}) |
| 435 |
if string(r.Extra) != "BC\x02\x00"+blEnc+"extra" { |
| 436 |
t.Errorf("extra is %q, want %q", r.Extra, "BC\x02\x00"+blEnc+"extra") |
| 437 |
} |
| 438 |
if r.ModTime.Unix() != 1e8 { |
| 439 |
t.Errorf("mtime is %d, want %d", r.ModTime.Unix(), uint32(1e8)) |
| 440 |
} |
| 441 |
if r.Name != "name" { |
| 442 |
t.Errorf("name is %q, want %q", r.Name, "name") |
| 443 |
} |
| 444 |
bl = ExpectedMemberSize(r.Header) |
| 445 |
if bl != wbl[0] { |
| 446 |
t.Errorf("expectedMemberSize is %d, want %d", bl, wbl[0]) |
| 447 |
} |
| 448 |
b = make([]byte, len("payload1payloadTwo")+1) |
| 449 |
n, err = r.Read(b) |
| 450 |
if err != io.EOF { |
| 451 |
t.Errorf("Read: %v", err) |
| 452 |
} |
| 453 |
if bl := ExpectedMemberSize(r.Header); bl != len(MagicBlock) { |
| 454 |
t.Errorf("expectedMemberSize is %d, want %d", bl, len(MagicBlock)) |
| 455 |
} |
| 456 |
if string(r.Extra) != "BC\x02\x00\x1b\x00" { |
| 457 |
t.Errorf("extra is %q, want %q", r.Extra, "BC\x02\x00\x1b\x00") |
| 458 |
} |
| 459 |
if string(b[:n]) != "payload1payloadTwo" { |
| 460 |
t.Errorf("payload is %q, want %q", string(b[:n]), "payload1payloadTwo") |
| 461 |
} |
| 462 |
if err := r.Seek(Offset{}); err != nil { |
| 463 |
t.Errorf("Seek: %v", err) |
| 464 |
} |
| 465 |
n, err = r.Read(b) |
| 466 |
if err != io.EOF { |
| 467 |
t.Errorf("Read: %v", err) |
| 468 |
} |
| 469 |
if string(b[:n]) != "payload1payloadTwo" { |
| 470 |
t.Errorf("payload is %q, want %q", string(b[:n]), "payload1payloadTwo") |
| 471 |
} |
| 472 |
if err := r.Seek(Offset{File: offset}); err != nil { |
| 473 |
t.Fatalf("Seek: %v", err) |
| 474 |
} |
| 475 |
bl = ExpectedMemberSize(r.Header) |
| 476 |
if bl != wbl[1] { |
| 477 |
t.Errorf("expectedMemberSize is %d, want %d", bl, wbl[1]) |
| 478 |
} |
| 479 |
b = make([]byte, bl+1) |
| 480 |
n, err = r.Read(b) |
| 481 |
if err != io.EOF { |
| 482 |
t.Errorf("Read: %v", err) |
| 483 |
} |
| 484 |
if err := r.Close(); err != nil { |
| 485 |
t.Errorf("r.Close: %v", err) |
| 486 |
} |
| 487 |
if err := f.Close(); err != nil { |
| 488 |
t.Errorf("f.Close: %v", err) |
| 489 |
} |
| 490 |
if string(b[:n]) != "payloadTwo" { |
| 491 |
t.Errorf("payload is %q, want %q", string(b[:n]), "payloadTwo") |
| 492 |
} |
| 493 |
} |
| 494 |
|
| 495 |
type errorReadSeeker struct { |
| 496 |
r io.ReadSeeker |
| 497 |
err error |
| 498 |
} |
| 499 |
|
| 500 |
func (r errorReadSeeker) Read(p []byte) (int, error) { |
| 501 |
n, err := r.r.Read(p) |
| 502 |
if err == nil && r.err != nil { |
| 503 |
err = r.err |
| 504 |
} |
| 505 |
return n, err |
| 506 |
} |
| 507 |
|
| 508 |
func (r errorReadSeeker) Seek(offset int64, whence int) (int64, error) { |
| 509 |
n, err := r.r.Seek(offset, whence) |
| 510 |
if r.err != nil { |
| 511 |
err = r.err |
| 512 |
} |
| 513 |
return n, err |
| 514 |
} |
| 515 |
|
| 516 |
func TestSeekErrorDeadlock(t *testing.T) { |
| 517 |
buf := new(bytes.Buffer) |
| 518 |
|
| 519 |
w := NewWriter(buf, *conc) |
| 520 |
w.Comment = "comment" |
| 521 |
w.Extra = []byte("extra") |
| 522 |
w.ModTime = time.Unix(1e8, 0) |
| 523 |
w.Name = "name" |
| 524 |
if _, err := w.Write([]byte("payload")); err != nil { |
| 525 |
t.Fatalf("Write: %v", err) |
| 526 |
} |
| 527 |
if err := w.Close(); err != nil { |
| 528 |
t.Fatalf("Writer.Close: %v", err) |
| 529 |
} |
| 530 |
e := &errorReadSeeker{r: bytes.NewReader(buf.Bytes())} |
| 531 |
r, err := NewReader(e, *conc) |
| 532 |
if err != nil { |
| 533 |
t.Fatalf("NewReader: %v", err) |
| 534 |
} |
| 535 |
r.Seek(Offset{File: 0}) |
| 536 |
e.err = errors.New("bad seek error") |
| 537 |
err = r.Seek(Offset{File: 1}) |
| 538 |
if err == nil { |
| 539 |
t.Error("Expected error.", err) |
| 540 |
} |
| 541 |
r.Close() |
| 542 |
} |
| 543 |
|
| 544 |
type countReadSeeker struct { |
| 545 |
mu sync.Mutex |
| 546 |
r io.ReadSeeker |
| 547 |
_didSeek bool |
| 548 |
n int64 |
| 549 |
} |
| 550 |
|
| 551 |
func (r *countReadSeeker) offset() int64 { |
| 552 |
r.mu.Lock() |
| 553 |
defer r.mu.Unlock() |
| 554 |
|
| 555 |
return r.n |
| 556 |
} |
| 557 |
|
| 558 |
func (r *countReadSeeker) didSeek() bool { |
| 559 |
r.mu.Lock() |
| 560 |
defer r.mu.Unlock() |
| 561 |
|
| 562 |
return r._didSeek |
| 563 |
} |
| 564 |
|
| 565 |
func (r *countReadSeeker) Read(p []byte) (int, error) { |
| 566 |
r.mu.Lock() |
| 567 |
defer r.mu.Unlock() |
| 568 |
|
| 569 |
r._didSeek = false |
| 570 |
n, err := r.r.Read(p) |
| 571 |
r.n += int64(n) |
| 572 |
return n, err |
| 573 |
} |
| 574 |
|
| 575 |
func (r *countReadSeeker) Seek(offset int64, whence int) (int64, error) { |
| 576 |
r.mu.Lock() |
| 577 |
defer r.mu.Unlock() |
| 578 |
|
| 579 |
r._didSeek = true |
| 580 |
return r.r.Seek(offset, whence) |
| 581 |
} |
| 582 |
|
| 583 |
func TestSeekFast(t *testing.T) { |
| 584 |
// Under these conditions we cannot guarantee that a worker |
| 585 |
// will not read bytes after a Seek call has been made. |
| 586 |
if *conc != 1 && runtime.GOMAXPROCS(0) > 1 { |
| 587 |
return |
| 588 |
} |
| 589 |
const ( |
| 590 |
infix = "payload" |
| 591 |
blocks = 10 |
| 592 |
) |
| 593 |
|
| 594 |
// Use different caches. |
| 595 |
for _, cache := range []Cache{ |
| 596 |
nil, // Explicitly nil. |
| 597 |
|
| 598 |
cache.NewLRU(0), // Functionally nil. |
| 599 |
cache.NewLRU(1), |
| 600 |
cache.NewLRU(blocks / 2), |
| 601 |
cache.NewLRU(blocks), |
| 602 |
cache.NewLRU(blocks + 1), |
| 603 |
|
| 604 |
cache.NewRandom(0), // Functionally nil. |
| 605 |
cache.NewRandom(1), |
| 606 |
cache.NewRandom(blocks / 2), |
| 607 |
cache.NewRandom(blocks), |
| 608 |
cache.NewRandom(blocks + 1), |
| 609 |
} { |
| 610 |
var ( |
| 611 |
buf bytes.Buffer |
| 612 |
offsets = []int{0} |
| 613 |
) |
| 614 |
w := NewWriter(&buf, 1) |
| 615 |
for i := 0; i < blocks; i++ { |
| 616 |
if _, err := fmt.Fprintf(w, "%d%[2]s%[1]d", i, infix); err != nil { |
| 617 |
t.Fatalf("Write: %v", err) |
| 618 |
} |
| 619 |
if err := w.Flush(); err != nil { |
| 620 |
t.Fatalf("Flush: %v", err) |
| 621 |
} |
| 622 |
if err := w.Wait(); err != nil { |
| 623 |
t.Fatalf("Wait: %v", err) |
| 624 |
} |
| 625 |
offsets = append(offsets, buf.Len()) |
| 626 |
} |
| 627 |
w.Close() |
| 628 |
offsets = offsets[:len(offsets)-1] |
| 629 |
|
| 630 |
c := &countReadSeeker{r: bytes.NewReader(buf.Bytes())} |
| 631 |
|
| 632 |
// Insert a HasEOF to ensure it does not corrupt subsequent reads. |
| 633 |
HasEOF(bytes.NewReader(buf.Bytes())) |
| 634 |
|
| 635 |
r, err := NewReader(c, *conc) |
| 636 |
if err != nil { |
| 637 |
t.Fatalf("NewReader: %v", err) |
| 638 |
} |
| 639 |
|
| 640 |
r.SetCache(cache) |
| 641 |
p := make([]byte, len(infix)+2) |
| 642 |
|
| 643 |
func() { |
| 644 |
defer func() { |
| 645 |
r := recover() |
| 646 |
if r != nil { |
| 647 |
t.Fatalf("Seek on unread reader panicked: %v", r) |
| 648 |
} |
| 649 |
}() |
| 650 |
err := r.Seek(Offset{}) |
| 651 |
if err != nil { |
| 652 |
t.Fatalf("Seek: %v", err) |
| 653 |
} |
| 654 |
}() |
| 655 |
|
| 656 |
// Standard read through of the data. |
| 657 |
for i := range offsets { |
| 658 |
n, err := r.Read(p) |
| 659 |
if n != len(p) { |
| 660 |
t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p)) |
| 661 |
} |
| 662 |
if err != nil { |
| 663 |
t.Fatalf("Read: %v", err) |
| 664 |
} |
| 665 |
got := string(p) |
| 666 |
want := fmt.Sprintf("%d%[2]s%[1]d", i, infix) |
| 667 |
if got != want { |
| 668 |
t.Errorf("Unexpected result: got:%q want:%q", got, want) |
| 669 |
} |
| 670 |
} |
| 671 |
|
| 672 |
// Seek to each block in turn |
| 673 |
for i, o := range offsets { |
| 674 |
err := r.Seek(Offset{File: int64(o)}) |
| 675 |
if err != nil { |
| 676 |
t.Fatalf("Seek: %v", err) |
| 677 |
} |
| 678 |
n, err := r.Read(p) |
| 679 |
if n != len(p) { |
| 680 |
t.Errorf("Unexpected read length: got:%d want:%d", n, len(p)) |
| 681 |
} |
| 682 |
if err != nil { |
| 683 |
t.Fatalf("Read: %v", err) |
| 684 |
} |
| 685 |
got := string(p) |
| 686 |
want := fmt.Sprintf("%d%[2]s%[1]d", i, infix) |
| 687 |
if got != want { |
| 688 |
t.Errorf("Unexpected result: got:%q want:%q", got, want) |
| 689 |
} |
| 690 |
} |
| 691 |
|
| 692 |
// Seek to each block in turn, but read the infix and then the first 2 bytes. |
| 693 |
for i, o := range offsets { |
| 694 |
if err := r.Seek(Offset{File: int64(o), Block: 1}); err != nil { |
| 695 |
t.Fatalf("Seek: %v", err) |
| 696 |
} |
| 697 |
p = p[:len(infix)] |
| 698 |
n, err := r.Read(p) |
| 699 |
if n != len(p) { |
| 700 |
t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p)) |
| 701 |
} |
| 702 |
if err != nil { |
| 703 |
t.Fatalf("Read: %v", err) |
| 704 |
} |
| 705 |
got := string(p) |
| 706 |
want := infix |
| 707 |
if got != want { |
| 708 |
t.Fatalf("Unexpected result: got:%q want:%q", got, want) |
| 709 |
} |
| 710 |
|
| 711 |
// Check whether the underlying reader was seeked or read. |
| 712 |
hasRead := c.offset() |
| 713 |
if err = r.Seek(Offset{File: int64(o), Block: 0}); err != nil { |
| 714 |
t.Fatalf("Seek: %v", err) |
| 715 |
} |
| 716 |
if b := c.offset() - hasRead; b != 0 { |
| 717 |
t.Errorf("Seek performed unexpected read: %d bytes", b) |
| 718 |
} |
| 719 |
if c.didSeek() { |
| 720 |
t.Error("Seek caused underlying Seek.") |
| 721 |
} |
| 722 |
|
| 723 |
p = p[:2] |
| 724 |
n, err = r.Read(p) |
| 725 |
if n != len(p) { |
| 726 |
t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p)) |
| 727 |
} |
| 728 |
if err != nil { |
| 729 |
t.Fatalf("Read: %v", err) |
| 730 |
} |
| 731 |
got = string(p) |
| 732 |
want = fmt.Sprintf("%dp", i) |
| 733 |
if got != want { |
| 734 |
t.Fatalf("Unexpected result: got:%q want:%q", got, want) |
| 735 |
} |
| 736 |
} |
| 737 |
r.Close() |
| 738 |
} |
| 739 |
} |
| 740 |
|
| 741 |
func TestCache(t *testing.T) { |
| 742 |
// Under these conditions we cannot guarantee that the order of |
| 743 |
// blocks returned by nextBlock work will not result in additional |
| 744 |
// cache puts. |
| 745 |
if *conc != 1 { |
| 746 |
return |
| 747 |
} |
| 748 |
const ( |
| 749 |
infix = "payload" |
| 750 |
blocks = 10 |
| 751 |
) |
| 752 |
|
| 753 |
// Each pattern is a series of seek-and-read (when the element >= 0) |
| 754 |
// or read (when the element < 0). Each read is exactly one block |
| 755 |
// worth of data. |
| 756 |
type opPair struct{ seekBlock, blockID int } |
| 757 |
patterns := []struct { |
| 758 |
ops []opPair |
| 759 |
|
| 760 |
// One for each cache case below. If new caches are added to the |
| 761 |
// test list, stats must be added here. |
| 762 |
expectedStats []cache.Stats |
| 763 |
}{ |
| 764 |
{ |
| 765 |
ops: []opPair{ |
| 766 |
{seekBlock: -1, blockID: 0}, |
| 767 |
{seekBlock: -1, blockID: 1}, |
| 768 |
{seekBlock: -1, blockID: 2}, |
| 769 |
{seekBlock: +0, blockID: 0}, |
| 770 |
{seekBlock: -1, blockID: 1}, |
| 771 |
{seekBlock: -1, blockID: 2}, |
| 772 |
{seekBlock: -1, blockID: 3}, |
| 773 |
{seekBlock: -1, blockID: 4}, |
| 774 |
}, |
| 775 |
expectedStats: []cache.Stats{ |
| 776 |
{}, // nil cache. |
| 777 |
{}, // nil cache: LRU(0) |
| 778 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1) |
| 779 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(5) |
| 780 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10) |
| 781 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11) |
| 782 |
{}, // nil cache: FIFO(0) |
| 783 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1) |
| 784 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(5) |
| 785 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(10) |
| 786 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(11) |
| 787 |
{}, // nil cache: Random(0) |
| 788 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1) |
| 789 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(5) |
| 790 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(10) |
| 791 |
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(11) |
| 792 |
}, |
| 793 |
}, |
| 794 |
{ |
| 795 |
ops: []opPair{ |
| 796 |
{seekBlock: -1, blockID: 0}, |
| 797 |
{seekBlock: -1, blockID: 1}, |
| 798 |
{seekBlock: -1, blockID: 2}, |
| 799 |
{seekBlock: +1, blockID: 1}, |
| 800 |
{seekBlock: -1, blockID: 2}, |
| 801 |
{seekBlock: -1, blockID: 3}, |
| 802 |
{seekBlock: -1, blockID: 4}, |
| 803 |
{seekBlock: -1, blockID: 5}, |
| 804 |
}, |
| 805 |
expectedStats: []cache.Stats{ |
| 806 |
{}, // nil cache. |
| 807 |
{}, // nil cache. |
| 808 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 4}, // LRU(1) |
| 809 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(5) |
| 810 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10) |
| 811 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11) |
| 812 |
{}, // nil cache: FIFO(0) |
| 813 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1) |
| 814 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(5) |
| 815 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(10) |
| 816 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(11) |
| 817 |
{}, // nil cache: Random(0) |
| 818 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 4}, // Random(1) |
| 819 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(5) |
| 820 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(10) |
| 821 |
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(11) |
| 822 |
}, |
| 823 |
}, |
| 824 |
{ |
| 825 |
ops: []opPair{ |
| 826 |
{seekBlock: -1, blockID: 0}, |
| 827 |
{seekBlock: -1, blockID: 1}, |
| 828 |
{seekBlock: -1, blockID: 2}, |
| 829 |
{seekBlock: +2, blockID: 2}, |
| 830 |
{seekBlock: -1, blockID: 3}, |
| 831 |
{seekBlock: -1, blockID: 4}, |
| 832 |
{seekBlock: -1, blockID: 5}, |
| 833 |
{seekBlock: -1, blockID: 6}, |
| 834 |
}, |
| 835 |
// Re-reading the same block avoids a cache look-up. |
| 836 |
expectedStats: []cache.Stats{ |
| 837 |
{}, // nil cache. |
| 838 |
{}, // nil cache. |
| 839 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // LRU(1) |
| 840 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // LRU(5) |
| 841 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // LRU(10) |
| 842 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // LRU(11) |
| 843 |
{}, // nil cache: FIFO(0) |
| 844 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // FIFO(1) |
| 845 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // FIFO(5) |
| 846 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // FIFO(10) |
| 847 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // FIFO(11) |
| 848 |
{}, // nil cache: Random(0) |
| 849 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // Random(1) |
| 850 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // Random(5) |
| 851 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // Random(10) |
| 852 |
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // Random(11) |
| 853 |
}, |
| 854 |
}, |
| 855 |
{ |
| 856 |
ops: []opPair{ |
| 857 |
{seekBlock: -1, blockID: 0}, |
| 858 |
{seekBlock: -1, blockID: 1}, |
| 859 |
{seekBlock: -1, blockID: 2}, |
| 860 |
{seekBlock: +3, blockID: 3}, |
| 861 |
{seekBlock: -1, blockID: 4}, |
| 862 |
{seekBlock: -1, blockID: 5}, |
| 863 |
{seekBlock: -1, blockID: 6}, |
| 864 |
{seekBlock: -1, blockID: 7}, |
| 865 |
}, |
| 866 |
expectedStats: []cache.Stats{ |
| 867 |
{}, // nil cache. |
| 868 |
{}, // nil cache. |
| 869 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1) |
| 870 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // LRU(5) |
| 871 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10) |
| 872 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11) |
| 873 |
{}, // nil cache: FIFO(0) |
| 874 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1) |
| 875 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // FIFO(5) |
| 876 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(10) |
| 877 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(11) |
| 878 |
{}, // nil cache: Random(0) |
| 879 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1) |
| 880 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // Random(5) |
| 881 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(10) |
| 882 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(11) |
| 883 |
}, |
| 884 |
}, |
| 885 |
{ |
| 886 |
ops: []opPair{ |
| 887 |
{seekBlock: -1, blockID: 0}, |
| 888 |
{seekBlock: -1, blockID: 1}, |
| 889 |
{seekBlock: -1, blockID: 2}, |
| 890 |
{seekBlock: +4, blockID: 4}, |
| 891 |
{seekBlock: -1, blockID: 5}, |
| 892 |
{seekBlock: -1, blockID: 6}, |
| 893 |
{seekBlock: -1, blockID: 7}, |
| 894 |
{seekBlock: -1, blockID: 8}, |
| 895 |
}, |
| 896 |
expectedStats: []cache.Stats{ |
| 897 |
{}, // nil cache. |
| 898 |
{}, // nil cache. |
| 899 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1) |
| 900 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // LRU(5) |
| 901 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10) |
| 902 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11) |
| 903 |
{}, // nil cache: FIFO(0) |
| 904 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1) |
| 905 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // FIFO(5) |
| 906 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(10) |
| 907 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(11) |
| 908 |
{}, // nil cache: Random(0) |
| 909 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1) |
| 910 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // Random(5) |
| 911 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(10) |
| 912 |
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(11) |
| 913 |
}, |
| 914 |
}, |
| 915 |
{ |
| 916 |
ops: []opPair{ |
| 917 |
{seekBlock: -1, blockID: 0}, |
| 918 |
{seekBlock: -1, blockID: 1}, |
| 919 |
{seekBlock: -1, blockID: 2}, |
| 920 |
{seekBlock: +1, blockID: 1}, |
| 921 |
{seekBlock: +2, blockID: 2}, |
| 922 |
{seekBlock: +1, blockID: 1}, |
| 923 |
{seekBlock: +1, blockID: 1}, |
| 924 |
{seekBlock: -1, blockID: 2}, |
| 925 |
{seekBlock: +7, blockID: 7}, |
| 926 |
{seekBlock: -1, blockID: 8}, |
| 927 |
{seekBlock: -1, blockID: 9}, |
| 928 |
}, |
| 929 |
expectedStats: []cache.Stats{ |
| 930 |
{}, // nil cache. |
| 931 |
{}, // nil cache. |
| 932 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 4}, // LRU(1) |
| 933 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(5) |
| 934 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(10) |
| 935 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(11) |
| 936 |
{}, // nil cache: FIFO(0) |
| 937 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 8}, // FIFO(1) |
| 938 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(5) |
| 939 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(10) |
| 940 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(11) |
| 941 |
{}, // nil cache: Random(0) |
| 942 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 4}, // Random(1) |
| 943 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(5) |
| 944 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(10) |
| 945 |
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(11) |
| 946 |
}, |
| 947 |
}, |
| 948 |
} |
| 949 |
|
| 950 |
for k, pat := range patterns { |
| 951 |
// Use different caches. |
| 952 |
for j, s := range []Cache{ |
| 953 |
nil, // Explicitly nil. |
| 954 |
|
| 955 |
cache.NewLRU(0), // Functionally nil. |
| 956 |
cache.NewLRU(1), |
| 957 |
cache.NewLRU(blocks / 2), |
| 958 |
cache.NewLRU(blocks), |
| 959 |
cache.NewLRU(blocks + 1), |
| 960 |
|
| 961 |
cache.NewFIFO(0), // Functionally nil. |
| 962 |
cache.NewFIFO(1), |
| 963 |
cache.NewFIFO(blocks / 2), |
| 964 |
cache.NewFIFO(blocks), |
| 965 |
cache.NewFIFO(blocks + 1), |
| 966 |
|
| 967 |
cache.NewRandom(0), // Functionally nil. |
| 968 |
cache.NewRandom(1), |
| 969 |
cache.NewRandom(blocks / 2), |
| 970 |
cache.NewRandom(blocks), |
| 971 |
cache.NewRandom(blocks + 1), |
| 972 |
} { |
| 973 |
var ( |
| 974 |
buf bytes.Buffer |
| 975 |
offsets = []int{0} |
| 976 |
) |
| 977 |
w := NewWriter(&buf, 1) |
| 978 |
for i := 0; i < blocks; i++ { |
| 979 |
if _, err := fmt.Fprintf(w, "%d%[2]s%[1]d", i, infix); err != nil { |
| 980 |
t.Fatalf("Write: %v", err) |
| 981 |
} |
| 982 |
if err := w.Flush(); err != nil { |
| 983 |
t.Fatalf("Flush: %v", err) |
| 984 |
} |
| 985 |
if err := w.Wait(); err != nil { |
| 986 |
t.Fatalf("Wait: %v", err) |
| 987 |
} |
| 988 |
offsets = append(offsets, buf.Len()) |
| 989 |
} |
| 990 |
w.Close() |
| 991 |
offsets = offsets[:len(offsets)-1] |
| 992 |
|
| 993 |
br := bytes.NewReader(buf.Bytes()) |
| 994 |
// Insert a HasEOF to ensure it does not corrupt subsequent reads. |
| 995 |
HasEOF(br) |
| 996 |
|
| 997 |
r, err := NewReader(br, *conc) |
| 998 |
if err != nil { |
| 999 |
t.Fatalf("NewReader: %v", err) |
| 1000 |
} |
| 1001 |
var stats *cache.StatsRecorder |
| 1002 |
if s != nil { |
| 1003 |
stats = &cache.StatsRecorder{Cache: s} |
| 1004 |
s = stats |
| 1005 |
} |
| 1006 |
r.SetCache(s) |
| 1007 |
p := make([]byte, len(infix)+2) |
| 1008 |
|
| 1009 |
for _, op := range pat.ops { |
| 1010 |
if op.seekBlock >= 0 { |
| 1011 |
err := r.Seek(Offset{File: int64(offsets[op.seekBlock])}) |
| 1012 |
if err != nil { |
| 1013 |
t.Fatalf("Seek: %v", err) |
| 1014 |
} |
| 1015 |
} |
| 1016 |
n, err := r.Read(p) |
| 1017 |
if n != len(p) { |
| 1018 |
t.Errorf("Unexpected read length: got:%d want:%d", n, len(p)) |
| 1019 |
} |
| 1020 |
if err != nil { |
| 1021 |
t.Fatalf("Read: %v", err) |
| 1022 |
} |
| 1023 |
got := string(p) |
| 1024 |
want := fmt.Sprintf("%d%[2]s%[1]d", op.blockID, infix) |
| 1025 |
if got != want { |
| 1026 |
t.Errorf("Unexpected result: got:%q want:%q", got, want) |
| 1027 |
} |
| 1028 |
} |
| 1029 |
if stats != nil && stats.Stats() != pat.expectedStats[j] { |
| 1030 |
t.Errorf("Unexpected result for cache %d pattern %d: got:%+v want:%+v", j, k, stats.Stats(), pat.expectedStats[j]) |
| 1031 |
} |
| 1032 |
r.Close() |
| 1033 |
} |
| 1034 |
} |
| 1035 |
} |
| 1036 |
|
| 1037 |
func TestBlocked(t *testing.T) { |
| 1038 |
const ( |
| 1039 |
infix = "payload" |
| 1040 |
blocks = 10 |
| 1041 |
) |
| 1042 |
|
| 1043 |
for _, blocked := range []bool{false, true} { |
| 1044 |
var ( |
| 1045 |
buf bytes.Buffer |
| 1046 |
want bytes.Buffer |
| 1047 |
) |
| 1048 |
w := NewWriter(&buf, 1) |
| 1049 |
for i := 0; i < blocks; i++ { |
| 1050 |
if _, err := fmt.Fprintf(w, "%d%[2]s%[1]d\n", i, infix); err != nil { |
| 1051 |
t.Fatalf("Write: %v", err) |
| 1052 |
} |
| 1053 |
if err := w.Flush(); err != nil { |
| 1054 |
t.Fatalf("Flush: %v", err) |
| 1055 |
} |
| 1056 |
if _, err := fmt.Fprintf(&want, "%d%[2]s%[1]d\n", i, infix); err != nil { |
| 1057 |
t.Fatalf("Write: %v", err) |
| 1058 |
} |
| 1059 |
} |
| 1060 |
err := w.Close() |
| 1061 |
if err != nil { |
| 1062 |
t.Fatalf("unexpected error on Close: %v", err) |
| 1063 |
} |
| 1064 |
|
| 1065 |
r, err := NewReader(bytes.NewReader(buf.Bytes()), *conc) |
| 1066 |
if err != nil { |
| 1067 |
t.Fatalf("NewReader: %v", err) |
| 1068 |
} |
| 1069 |
r.Blocked = blocked |
| 1070 |
|
| 1071 |
p := make([]byte, len(infix)) |
| 1072 |
var ( |
| 1073 |
got []byte |
| 1074 |
gotBlocks int |
| 1075 |
) |
| 1076 |
for { |
| 1077 |
n, err := r.Read(p) |
| 1078 |
got = append(got, p[:n]...) |
| 1079 |
if err != nil { |
| 1080 |
if err == io.EOF && n != 0 { |
| 1081 |
gotBlocks++ |
| 1082 |
continue |
| 1083 |
} |
| 1084 |
break |
| 1085 |
} |
| 1086 |
} |
| 1087 |
if !blocked && gotBlocks != 1 { |
| 1088 |
t.Errorf("unexpected number of blocks:\n\tgot:%d\n\twant:%d", gotBlocks, 1) |
| 1089 |
} |
| 1090 |
if blocked && gotBlocks != blocks { |
| 1091 |
t.Errorf("unexpected number of blocks:\n\tgot:%d\n\twant:%d", gotBlocks, blocks) |
| 1092 |
} |
| 1093 |
if !bytes.Equal(got, want.Bytes()) { |
| 1094 |
t.Errorf("unexpected result:\n\tgot:%q\n\twant:%q", got, want.Bytes()) |
| 1095 |
} |
| 1096 |
r.Close() |
| 1097 |
} |
| 1098 |
} |
| 1099 |
|
| 1100 |
var fuzzCrashers = []string{ |
| 1101 |
// Invalid block size. |
| 1102 |
"\x1f\x8b\bu0000000\x0000000000" + |
| 1103 |
"000000000BC\x02\x000\x0000000" + |
| 1104 |
"00000000000000000000" + |
| 1105 |
"00000000000000000000" + |
| 1106 |
"000000000000000000\x00", |
| 1107 |
"\x1f\x8b\b\xc4000000V\x0000000000" + |
| 1108 |
"00000000000000000000" + |
| 1109 |
"00000000000000000000" + |
| 1110 |
"00000000000000000000" + |
| 1111 |
"000000000000BC\x02\x00w\x00\x030" + |
| 1112 |
"\x00\x00\x00\x00\x00\x00\x00\x00\x1f\x8b\bu000000\x00\x00" + |
| 1113 |
"\x1f\x8b\bu000000\b\x00BC\x02\x00\x00\x0000" + |
| 1114 |
"\x00", |
| 1115 |
|
| 1116 |
// Zero block size. |
| 1117 |
"\x1f\x8b\bu000000V\x0000000000" + |
| 1118 |
"00000000000000000000" + |
| 1119 |
"00000000000000000000" + |
| 1120 |
"00000000000000000000" + |
| 1121 |
"000000000000BC\x02\x00k\x0000" + |
| 1122 |
"0000000\x00", |
| 1123 |
"\x1f\x8b\bu\xe8k\x15k\x00sV\x00bcdefghi" + |
| 1124 |
"jklmnxpq\xc49\xbf\x1f\x8b\x0f\a/\x85\xba\xb0Q" + |
| 1125 |
"\xef (\x01\xbd\xbf\xefrde\a/\x85fghmjt\x00" + |
| 1126 |
"\xff\x00\x00v\x97x\x92zB\x80\x00142261869" + |
| 1127 |
"48039093abxdBC\x02\x00i\x00sV" + |
| 1128 |
"\xbbghmj\x00\x00", |
| 1129 |
} |
| 1130 |
|
| 1131 |
func TestFuzzCrashers(t *testing.T) { |
| 1132 |
for i, test := range fuzzCrashers { |
| 1133 |
func() { |
| 1134 |
i := i |
| 1135 |
defer func() { |
| 1136 |
r := recover() |
| 1137 |
if r != nil { |
| 1138 |
t.Errorf("unexpected panic for crasher %d: %v", i, r) |
| 1139 |
} |
| 1140 |
}() |
| 1141 |
r, err := NewReader(strings.NewReader(test), 0) |
| 1142 |
switch err { |
| 1143 |
case nil: |
| 1144 |
// Pass through. |
| 1145 |
case io.EOF, io.ErrUnexpectedEOF, ErrCorrupt: |
| 1146 |
return |
| 1147 |
default: |
| 1148 |
t.Fatalf("unexpected error creating reader: %v", err) |
| 1149 |
} |
| 1150 |
tmp := make([]byte, 1024) |
| 1151 |
for { |
| 1152 |
_, err := r.Read(tmp) |
| 1153 |
if err != nil { |
| 1154 |
break |
| 1155 |
} |
| 1156 |
} |
| 1157 |
}() |
| 1158 |
} |
| 1159 |
} |
| 1160 |
|
| 1161 |
func TestZeroNonZero(t *testing.T) { |
| 1162 |
const wrote = "second block" |
| 1163 |
buf := bytes.NewBuffer([]byte(MagicBlock)) |
| 1164 |
w := NewWriter(buf, 1) |
| 1165 |
_, err := w.Write([]byte(wrote)) |
| 1166 |
if err != nil { |
| 1167 |
w.Close() |
| 1168 |
t.Fatalf("unexpected error writing second block: %v", err) |
| 1169 |
} |
| 1170 |
err = w.Close() |
| 1171 |
if err != nil { |
| 1172 |
t.Fatalf("unexpected error closing writer: %v", err) |
| 1173 |
} |
| 1174 |
r, err := NewReader(buf, 1) |
| 1175 |
if err != nil { |
| 1176 |
t.Fatalf("unexpected error opening reader: %v", err) |
| 1177 |
} |
| 1178 |
defer r.Close() |
| 1179 |
var b [1024]byte |
| 1180 |
var got []byte |
| 1181 |
for { |
| 1182 |
n, err := r.Read(b[:]) |
| 1183 |
got = append(got, b[:n]...) |
| 1184 |
if err != nil { |
| 1185 |
break |
| 1186 |
} |
| 1187 |
} |
| 1188 |
if string(got) != wrote { |
| 1189 |
t.Errorf("unexpected round trip: got:%q want:%q", got, wrote) |
| 1190 |
} |
| 1191 |
} |
| 1192 |
|
| 1193 |
type zero struct{} |
| 1194 |
|
| 1195 |
func (z zero) Read(p []byte) (int, error) { |
| 1196 |
for i := range p { |
| 1197 |
p[i] = 0 |
| 1198 |
} |
| 1199 |
return len(p), nil |
| 1200 |
} |
| 1201 |
|
| 1202 |
func TestWriteByteCount(t *testing.T) { |
| 1203 |
cw, _ := NewWriterLevel(io.Discard, gzip.BestCompression, 4) |
| 1204 |
defer cw.Close() |
| 1205 |
n, err := io.Copy(cw, &io.LimitedReader{R: new(zero), N: 100000}) |
| 1206 |
if n != 100000 { |
| 1207 |
t.Errorf("Unexpected number of bytes, got:%d, want:%d", n, 100000) |
| 1208 |
} |
| 1209 |
if err != nil { |
| 1210 |
t.Errorf("Unexpected error got:%v", err) |
| 1211 |
} |
| 1212 |
} |
| 1213 |
|
| 1214 |
func TestSeekCacheReadahead(t *testing.T) { |
| 1215 |
// Check that we see the correct behavior when seeking to the block at the |
| 1216 |
// head of the readahead queue with caching and readahead enabled. |
| 1217 |
// See https://github.com/biogo/hts/issues/159. |
| 1218 |
fh, err := os.Open("testdata/dbscSNV_sample.gz") |
| 1219 |
if err != nil { |
| 1220 |
t.Fatalf("Open() error:%v", err) |
| 1221 |
} |
| 1222 |
rd, err := NewReader(fh, 2) |
| 1223 |
if err != nil { |
| 1224 |
t.Fatalf("NewReader() error:%v", err) |
| 1225 |
} |
| 1226 |
rd.SetCache(cache.NewLRU(32)) |
| 1227 |
buf := make([]byte, 2) |
| 1228 |
// Wait for readahead. |
| 1229 |
time.Sleep(100 * time.Millisecond) |
| 1230 |
// Seek to the beginning of the second block. |
| 1231 |
off := Offset{File: 9395, Block: 0} |
| 1232 |
err = rd.Seek(off) |
| 1233 |
if err != nil { |
| 1234 |
t.Fatalf("Seek() error:%v", err) |
| 1235 |
} |
| 1236 |
_, err = rd.Read(buf) |
| 1237 |
if err != nil { |
| 1238 |
t.Fatalf("Read() error:%v", err) |
| 1239 |
} |
| 1240 |
if !bytes.Equal(buf, []byte{'O', 'X'}) { |
| 1241 |
t.Errorf("Expected 'OX', got '%s'", buf) |
| 1242 |
} |
| 1243 |
if rd.LastChunk().Begin != off { |
| 1244 |
t.Errorf("Incorrect position: expected %+v, got %+v", off, rd.LastChunk().Begin) |
| 1245 |
} |
| 1246 |
} |
| 1247 |
|
| 1248 |
func BenchmarkWrite(b *testing.B) { |
| 1249 |
bg := NewWriter(io.Discard, *conc) |
| 1250 |
block := bytes.Repeat([]byte("repeated"), 50) |
| 1251 |
for i := 0; i < b.N; i++ { |
| 1252 |
for j := 0; j < 1000000; j++ { |
| 1253 |
bg.Write(block) |
| 1254 |
} |
| 1255 |
bg.Wait() |
| 1256 |
} |
| 1257 |
} |
| 1258 |
|
| 1259 |
func BenchmarkRead(b *testing.B) { |
| 1260 |
if *file == "" { |
| 1261 |
b.Skip("no bgzf file specified") |
| 1262 |
} |
| 1263 |
f, err := os.Open(*file) |
| 1264 |
if err != nil { |
| 1265 |
b.Fatalf("file open failed: %v", err) |
| 1266 |
} |
| 1267 |
defer f.Close() |
| 1268 |
|
| 1269 |
buf := make([]byte, 16384) |
| 1270 |
b.ResetTimer() |
| 1271 |
|
| 1272 |
for i := 0; i < b.N; i++ { |
| 1273 |
f.Seek(0, io.SeekStart) |
| 1274 |
bg, err := NewReader(f, *conc) |
| 1275 |
if err != nil { |
| 1276 |
b.Fatalf("bgzf open failed: %v", err) |
| 1277 |
} |
| 1278 |
for { |
| 1279 |
_, err = bg.Read(buf) |
| 1280 |
if err == io.EOF { |
| 1281 |
break |
| 1282 |
} |
| 1283 |
if err != nil { |
| 1284 |
b.Fatalf("bgzf read failed: %v", err) |
| 1285 |
} |
| 1286 |
} |
| 1287 |
bg.Close() |
| 1288 |
} |
| 1289 |
} |