Debian Codegraph @

storage/gcp/antispam/gcp_test.go

raw
1 // Copyright 2024 The Tessera authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 package gcp
16
17 import (
18 "crypto/sha256"
19 "os"
20 "testing"
21 "time"
22
23 "cloud.google.com/go/spanner/spannertest"
24 "github.com/transparency-dev/tessera"
25 "github.com/transparency-dev/tessera/api"
26 "github.com/transparency-dev/tessera/testonly"
27 "k8s.io/klog/v2"
28 )
29
30 type testLookup struct {
31 entryHash []byte
32 wantNotFound bool
33 }
34
35 func TestAntispamStorage(t *testing.T) {
36 for _, test := range []struct {
37 name string
38 opts AntispamOpts
39 logEntries [][]byte
40 lookupEntries []testLookup
41 }{
42 {
43 name: "roundtrip",
44 logEntries: [][]byte{
45 []byte("one"),
46 []byte("two"),
47 []byte("three"),
48 },
49 lookupEntries: []testLookup{
50 {
51 entryHash: testIDHash([]byte("one")),
52 }, {
53 entryHash: testIDHash([]byte("two")),
54 }, {
55 entryHash: testIDHash([]byte("three")),
56 }, {
57 entryHash: testIDHash([]byte("nowhere to be found")),
58 wantNotFound: true,
59 },
60 },
61 },
62 } {
63 t.Run(test.name, func(t *testing.T) {
64 closeDB := newSpannerDB(t)
65 defer closeDB()
66 as, err := NewAntispam(t.Context(), "projects/p/instances/i/databases/d", test.opts)
67 if err != nil {
68 t.Fatalf("NewAntispam: %v", err)
69 }
70
71 fl, shutdown := testonly.NewTestLog(t, tessera.NewAppendOptions().WithCheckpointInterval(time.Second))
72 defer func() {
73 if err := shutdown(t.Context()); err != nil {
74 t.Logf("shutdown: %v", err)
75 }
76 }()
77
78 f := as.Follower(testBundleHasher)
79
80 go f.Follow(t.Context(), fl.LogReader)
81
82 entryIndex := make(map[string]uint64)
83 a := tessera.NewPublicationAwaiter(t.Context(), fl.LogReader.ReadCheckpoint, 100*time.Millisecond)
84 for i, e := range test.logEntries {
85 entry := tessera.NewEntry(e)
86 f := fl.Appender.Add(t.Context(), entry)
87 idx, _, err := a.Await(t.Context(), f)
88 if err != nil {
89 t.Fatalf("Await(%d): %v", i, err)
90 }
91 klog.Infof("%d == %x", i, entry.Identity())
92 entryIndex[string(testIDHash(e))] = idx.Index
93 }
94
95 for {
96 time.Sleep(time.Second)
97 pos, err := f.EntriesProcessed(t.Context())
98 if err != nil {
99 t.Logf("EntriesProcessed: %v", err)
100 continue
101 }
102 sz, err := fl.LogReader.IntegratedSize(t.Context())
103 if err != nil {
104 t.Logf("IntegratedSize: %v", err)
105 continue
106 }
107 klog.Infof("Wait for follower (%d) to catch up with tree (%d)", pos, sz)
108 if pos >= sz {
109 break
110 }
111 }
112
113 for _, e := range test.lookupEntries {
114 gotIndex, err := as.index(t.Context(), e.entryHash)
115 if err != nil {
116 t.Errorf("error looking up hash %x: %v", e.entryHash, err)
117 }
118 wantIndex := entryIndex[string(e.entryHash)]
119 if gotIndex == nil {
120 if !e.wantNotFound {
121 t.Errorf("no index for hash %x, but expected index %d", e.entryHash, wantIndex)
122 }
123 continue
124 }
125 if *gotIndex != wantIndex {
126 t.Errorf("got index %d, want %d from looking up hash %x", gotIndex, wantIndex, e.entryHash)
127 }
128 }
129 })
130 }
131 }
132
133 func TestAntispamPushbackRecovers(t *testing.T) {
134 for _, test := range []struct {
135 name string
136 opts AntispamOpts
137 logEntries [][]byte
138 }{
139 {
140 name: "pushback",
141 opts: AntispamOpts{
142 PushbackThreshold: 1,
143 },
144 logEntries: [][]byte{
145 []byte("one"),
146 []byte("two"),
147 []byte("three"),
148 },
149 },
150 } {
151 t.Run(test.name, func(t *testing.T) {
152 closeDB := newSpannerDB(t)
153 defer closeDB()
154 as, err := NewAntispam(t.Context(), "projects/p/instances/i/databases/d", test.opts)
155 if err != nil {
156 t.Fatalf("NewAntispam: %v", err)
157 }
158
159 fl, shutdown := testonly.NewTestLog(t, tessera.NewAppendOptions().WithCheckpointInterval(time.Second))
160 defer func() {
161 if err := shutdown(t.Context()); err != nil {
162 t.Logf("shutdown: %v", err)
163 }
164 }()
165
166 f := as.Follower(testBundleHasher)
167
168 entryIndex := make(map[string]uint64)
169 a := tessera.NewPublicationAwaiter(t.Context(), fl.LogReader.ReadCheckpoint, 100*time.Millisecond)
170 for i, e := range test.logEntries {
171 entry := tessera.NewEntry(e)
172 f := fl.Appender.Add(t.Context(), entry)
173 idx, _, err := a.Await(t.Context(), f)
174 if err != nil {
175 t.Fatalf("Await(%d): %v", i, err)
176 }
177 klog.Infof("%d == %x", i, entry.Identity())
178 entryIndex[string(testIDHash(e))] = idx.Index
179 }
180
181 // Wait for entries te be integrated before we start the follower, so we know we'll hit the pushback condition
182 go f.Follow(t.Context(), fl.LogReader)
183
184 for {
185 time.Sleep(time.Second)
186 pos, err := f.EntriesProcessed(t.Context())
187 if err != nil {
188 t.Logf("EntriesProcessed: %v", err)
189 continue
190 }
191 sz, err := fl.LogReader.IntegratedSize(t.Context())
192 if err != nil {
193 t.Logf("IntegratedSize: %v", err)
194 continue
195 }
196 klog.Infof("Wait for follower (%d) to catch up with tree (%d)", pos, sz)
197 if pos >= sz {
198 break
199 }
200 }
201
202 // Ensure that the follower gets itself _out_ of pushback mode once it's caught up.
203 // We'll give the follower some time to do its thing and notice.
204 // It runs onces a second, so this should be plenty of time.
205 for i := range 5 {
206 time.Sleep(time.Second)
207 if !as.pushBack.Load() {
208 t.Logf("Antispam caught up and out of pushback in %ds", i)
209 return
210 }
211 }
212 t.Fatalf("pushBack remains true after 5 seconds despite being caught up!")
213 })
214 }
215 }
216
217 func newSpannerDB(t *testing.T) func() {
218 t.Helper()
219 srv, err := spannertest.NewServer("localhost:0")
220 if err != nil {
221 t.Fatalf("Failed to set up test spanner: %v", err)
222 }
223 if err := os.Setenv("SPANNER_EMULATOR_HOST", srv.Addr); err != nil {
224 t.Fatalf("Setenv: %v", err)
225 }
226 return srv.Close
227 }
228
229 func testIDHash(d []byte) []byte {
230 r := sha256.Sum256(d)
231 return r[:]
232 }
233
234 func testBundleHasher(b []byte) ([][]byte, error) {
235 bun := &api.EntryBundle{}
236 err := bun.UnmarshalText(b)
237 if err != nil {
238 return nil, err
239 }
240 r := make([][]byte, len(bun.Entries))
241 for i, e := range bun.Entries {
242 r[i] = testIDHash(e)
243 }
244 return r, err
245 }