Debian Codegraph @

maintnotifications/e2e/scenario_template.go.example

raw
1
2
3 package e2e
4
5 import (
6 "context"
7 "fmt"
8 "os"
9 "testing"
10 "time"
11
12 "github.com/redis/go-redis/v9"
13 "github.com/redis/go-redis/v9/hitless"
14 )
15
16 // TestScenarioTemplate is a template for writing scenario tests
17 // Copy this file and rename it to scenario_your_test_name.go
18 func TestScenarioTemplate(t *testing.T) {
19 if os.Getenv("E2E_SCENARIO_TESTS") != "true" {
20 t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true")
21 }
22
23 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
24 defer cancel()
25
26 // Step 1: Create client factory from configuration
27 factory, err := CreateTestClientFactory("enterprise-cluster") // or "standalone0"
28 if err != nil {
29 t.Fatalf("Failed to create client factory: %v", err)
30 }
31 defer factory.DestroyAll()
32
33 // Step 2: Create fault injector
34 faultInjector, err := CreateTestFaultInjector()
35 if err != nil {
36 t.Fatalf("Failed to create fault injector: %v", err)
37 }
38
39 // Step 3: Create Redis client with hitless upgrades
40 client, err := factory.Create("scenario-client", &CreateClientOptions{
41 Protocol: 3,
42 HitlessUpgradeConfig: &hitless.Config{
43 Mode: hitless.MaintNotificationsEnabled,
44 HandoffTimeout: 30000, // 30 seconds
45 RelaxedTimeout: 10000, // 10 seconds
46 MaxWorkers: 20,
47 },
48 ClientName: "scenario-test-client",
49 })
50 if err != nil {
51 t.Fatalf("Failed to create client: %v", err)
52 }
53
54 // Step 4: Verify initial connectivity
55 err = client.Ping(ctx).Err()
56 if err != nil {
57 t.Fatalf("Failed to ping Redis: %v", err)
58 }
59
60 t.Log("Initial setup completed successfully")
61
62 // Step 5: Start background operations (optional)
63 stopCh := make(chan struct{})
64 defer close(stopCh)
65
66 go func() {
67 counter := 0
68 ticker := time.NewTicker(100 * time.Millisecond)
69 defer ticker.Stop()
70
71 for {
72 select {
73 case <-stopCh:
74 return
75 case <-ticker.C:
76 key := fmt.Sprintf("test-key-%d", counter)
77 value := fmt.Sprintf("test-value-%d", counter)
78
79 err := client.Set(ctx, key, value, time.Minute).Err()
80 if err != nil {
81 t.Logf("Background operation failed: %v", err)
82 }
83
84 counter++
85 }
86 }
87 }()
88
89 // Step 6: Wait for baseline operations
90 time.Sleep(5 * time.Second)
91
92 // Step 7: Trigger fault injection scenario
93 t.Log("Triggering fault injection scenario...")
94
95 // Example: Cluster failover
96 // resp, err := faultInjector.TriggerClusterFailover(ctx, "node-1", false)
97 // if err != nil {
98 // t.Fatalf("Failed to trigger failover: %v", err)
99 // }
100
101 // Example: Network latency
102 // nodes := []string{"localhost:7001", "localhost:7002"}
103 // resp, err := faultInjector.SimulateNetworkLatency(ctx, nodes, 100*time.Millisecond, 20*time.Millisecond)
104 // if err != nil {
105 // t.Fatalf("Failed to simulate latency: %v", err)
106 // }
107
108 // Example: Complex sequence
109 // sequence := []SequenceAction{
110 // {
111 // Type: ActionNetworkLatency,
112 // Parameters: map[string]interface{}{
113 // "nodes": []string{"localhost:7001"},
114 // "latency": "50ms",
115 // },
116 // },
117 // {
118 // Type: ActionClusterFailover,
119 // Parameters: map[string]interface{}{
120 // "node_id": "node-1",
121 // "force": false,
122 // },
123 // Delay: 10 * time.Second,
124 // },
125 // }
126 // resp, err := faultInjector.ExecuteSequence(ctx, sequence)
127 // if err != nil {
128 // t.Fatalf("Failed to execute sequence: %v", err)
129 // }
130
131 // Step 8: Wait for fault injection to complete
132 // status, err := faultInjector.WaitForAction(ctx, resp.ActionID,
133 // WithMaxWaitTime(240*time.Second),
134 // WithPollInterval(2*time.Second))
135 // if err != nil {
136 // t.Fatalf("Fault injection failed: %v", err)
137 // }
138 // t.Logf("Fault injection completed: %s", status.Status)
139
140 // Step 9: Verify client remains operational during and after fault injection
141 time.Sleep(10 * time.Second)
142
143 err = client.Ping(ctx).Err()
144 if err != nil {
145 t.Errorf("Client not responsive after fault injection: %v", err)
146 }
147
148 // Step 10: Perform additional validation
149 testKey := "validation-key"
150 testValue := "validation-value"
151
152 err = client.Set(ctx, testKey, testValue, time.Minute).Err()
153 if err != nil {
154 t.Errorf("Failed to set validation key: %v", err)
155 }
156
157 retrievedValue, err := client.Get(ctx, testKey).Result()
158 if err != nil {
159 t.Errorf("Failed to get validation key: %v", err)
160 } else if retrievedValue != testValue {
161 t.Errorf("Validation failed: expected %s, got %s", testValue, retrievedValue)
162 }
163
164 t.Log("Scenario test completed successfully")
165 }
166
167 // Helper functions for common scenario patterns
168
169 func performContinuousOperations(ctx context.Context, client redis.UniversalClient, workerID int, stopCh <-chan struct{}, errorCh chan<- error) {
170 ticker := time.NewTicker(100 * time.Millisecond)
171 defer ticker.Stop()
172
173 counter := 0
174 for {
175 select {
176 case <-stopCh:
177 return
178 case <-ticker.C:
179 key := fmt.Sprintf("worker_%d_key_%d", workerID, counter)
180 value := fmt.Sprintf("value_%d", counter)
181
182 // Perform operation with timeout
183 opCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
184 err := client.Set(opCtx, key, value, time.Minute).Err()
185 cancel()
186
187 if err != nil {
188 select {
189 case errorCh <- err:
190 default:
191 }
192 }
193
194 counter++
195 }
196 }
197 }
198
199 func validateClusterHealth(ctx context.Context, client redis.UniversalClient) error {
200 // Basic connectivity test
201 if err := client.Ping(ctx).Err(); err != nil {
202 return fmt.Errorf("ping failed: %w", err)
203 }
204
205 // Test basic operations
206 testKey := "health-check-key"
207 testValue := "health-check-value"
208
209 if err := client.Set(ctx, testKey, testValue, time.Minute).Err(); err != nil {
210 return fmt.Errorf("set operation failed: %w", err)
211 }
212
213 retrievedValue, err := client.Get(ctx, testKey).Result()
214 if err != nil {
215 return fmt.Errorf("get operation failed: %w", err)
216 }
217
218 if retrievedValue != testValue {
219 return fmt.Errorf("value mismatch: expected %s, got %s", testValue, retrievedValue)
220 }
221
222 // Clean up
223 client.Del(ctx, testKey)
224
225 return nil
226 }
227
228 func waitForStableOperations(ctx context.Context, client redis.UniversalClient, duration time.Duration) error {
229 deadline := time.Now().Add(duration)
230 ticker := time.NewTicker(1 * time.Second)
231 defer ticker.Stop()
232
233 for time.Now().Before(deadline) {
234 select {
235 case <-ctx.Done():
236 return ctx.Err()
237 case <-ticker.C:
238 if err := validateClusterHealth(ctx, client); err != nil {
239 return fmt.Errorf("cluster health check failed: %w", err)
240 }
241 }
242 }
243
244 return nil
245 }