Debian Codegraph @

.github/workflows/integration-test.yml

raw
1 name: Integration Test
2
3 permissions:
4 contents: read
5
6 on:
7 schedule:
8 # Run daily at 2 AM UTC
9 - cron: '0 2 * * *'
10 workflow_dispatch:
11 inputs:
12 corpus_url:
13 description: 'URL to email corpus (zip file)'
14 required: false
15 default: 'https://github.com/rspamd/rspamd-test-corpus/releases/download/v1.0/rspamd-test-corpus.zip'
16
17 jobs:
18 integration-test:
19 name: Integration & Load Test
20 if: github.event_name != 'schedule' || github.repository == 'rspamd/rspamd'
21 runs-on: ubuntu-latest
22 timeout-minutes: 30
23
24 steps:
25 - name: Checkout code
26 uses: actions/checkout@v4
27
28 - name: Set up Python
29 uses: actions/setup-python@v5
30 with:
31 python-version: '3.11'
32
33 - name: Install system dependencies
34 run: |
35 sudo apt-get update
36 sudo apt-get install -y \
37 build-essential \
38 cmake \
39 ninja-build \
40 ragel \
41 libluajit-5.1-dev \
42 libglib2.0-dev \
43 libssl-dev \
44 libicu-dev \
45 libsodium-dev \
46 libhyperscan-dev \
47 libpcre2-dev \
48 libjemalloc-dev \
49 libunwind-dev \
50 libmagic-dev \
51 libarchive-dev \
52 libzstd-dev \
53 libbrotli-dev \
54 libfann-dev \
55 libstemmer-dev \
56 liblua5.1-dev \
57 redis-server \
58 sqlite3 \
59 libsqlite3-dev
60
61 - name: Install Python dependencies
62 run: |
63 pip install requests
64
65 - name: Build Rspamd
66 working-directory: .
67 run: |
68 mkdir -p build install
69 cd build
70 cmake -DCMAKE_INSTALL_PREFIX=/usr \
71 -DCONFDIR=/etc/rspamd \
72 -DENABLE_COVERAGE=OFF \
73 -DENABLE_FULL_DEBUG=ON \
74 -DSANITIZE=address,leak \
75 -GNinja ..
76 ninja
77 DESTDIR="${GITHUB_WORKSPACE}/install" ninja install
78
79 - name: Create static encryption keys
80 working-directory: test/integration
81 run: |
82 # Use static keys to avoid LD_LIBRARY_PATH issues with rspamadm
83 cat > .env.keys <<'EOF'
84 # Rspamd integration test keys (static for testing)
85
86 # Fuzzy worker keypair
87 RSPAMD_FUZZY_WORKER_PRIVKEY=aergrxbdfppyo8x3upyw5brui1sug93ztyd9gcuuan141iy8zi1y
88 RSPAMD_FUZZY_WORKER_PUBKEY=dsrea5fbqaqtzccno8roczeym5hgbokn5wxr5cyyhhzrqciiogpy
89
90 # Fuzzy check encryption key (same as fuzzy worker pubkey)
91 RSPAMD_FUZZY_ENCRYPTION_KEY=dsrea5fbqaqtzccno8roczeym5hgbokn5wxr5cyyhhzrqciiogpy
92
93 # Normal worker keypair (for encrypted inter-worker communication)
94 RSPAMD_WORKER_PRIVKEY=y8rq4a5rb7wj3xpefhx9t8gg3npfy4fswbwxyw7ma8hj46m36w1y
95 RSPAMD_WORKER_PUBKEY=bnw1ga6h7q155qei7tzz9k5z4rb9msbf8gwra4jbzsyj8chg5n3y
96
97 # Proxy worker keypair
98 RSPAMD_PROXY_PRIVKEY=ad9cz7mhb881ngsr3pncxydm9d6t917hyrysywixwpzntzhikruy
99 RSPAMD_PROXY_PUBKEY=bjkakz1r7zps7z774yn11gb9det5rw3dxb7z7559p5twzucneucy
100 EOF
101
102 echo "Static keys configured for testing"
103
104 - name: Download email corpus
105 working-directory: test/integration
106 run: |
107 # Use provided URL or default corpus from rspamd-test-corpus
108 CORPUS_URL="${{ github.event.inputs.corpus_url }}"
109 if [ -z "$CORPUS_URL" ]; then
110 # Default: use latest release from rspamd-test-corpus
111 CORPUS_URL="https://github.com/rspamd/rspamd-test-corpus/releases/latest/download/rspamd-test-corpus.zip"
112 fi
113
114 echo "Downloading corpus from: $CORPUS_URL"
115
116 # Create data directory for corpus
117 mkdir -p data
118 chmod 777 data
119
120 curl -L "$CORPUS_URL" -o data/corpus.zip
121
122 # Extract corpus
123 unzip data/corpus.zip -d data/
124
125 # The archive contains a 'corpus' directory, so we should have data/corpus/ now
126 ls -lh data/corpus/
127
128 - name: Start Docker Compose
129 working-directory: test/integration
130 run: |
131 docker compose up -d
132
133 # Wait for services to be ready
134 # Rspamd takes ~30-40s to compile TLD database and load maps
135 echo "Waiting for services to start..."
136 sleep 20
137
138 # Check services
139 echo "=== Docker Compose Services Status ==="
140 docker compose ps
141
142 echo ""
143 echo "=== Redis Logs ==="
144 docker compose logs redis
145
146 echo ""
147 echo "=== Rspamd Logs ==="
148 docker compose logs rspamd
149
150 - name: Wait for Rspamd to be ready
151 working-directory: test/integration
152 run: |
153 echo "Waiting for Rspamd services to initialize..."
154
155 # Wait for healthcheck to pass
156 for i in {1..60}; do
157 if curl -s http://localhost:50002/ping > /dev/null 2>&1; then
158 echo "✓ Rspamd Controller is ready!"
159
160 # Also check proxy
161 if curl -s http://localhost:50004/ping > /dev/null 2>&1; then
162 echo "✓ Rspamd Proxy is ready!"
163 else
164 echo "⚠ WARNING: Proxy not responding, but continuing..."
165 fi
166
167 # Show Rspamd stat
168 echo ""
169 echo "=== Rspamd Status ==="
170 curl -s http://localhost:50002/stat 2>/dev/null | head -20 || echo "Cannot get stat"
171
172 exit 0
173 fi
174 echo "Waiting for Rspamd... (attempt $i/60)"
175 sleep 3
176 done
177
178 echo "❌ Rspamd failed to start within timeout!"
179 echo ""
180 echo "=== Docker Compose Status ==="
181 docker compose ps
182 echo ""
183 echo "=== Full Rspamd logs ==="
184 docker compose logs rspamd
185 echo ""
186 echo "=== Checking for ASAN logs in container ==="
187 docker compose exec -T rspamd ls -la /data/ || true
188 docker compose exec -T rspamd cat /data/asan.log* 2>/dev/null || echo "No ASAN logs found"
189 echo ""
190 echo "=== Container stderr/stdout ==="
191 docker logs rspamd-main 2>&1 || true
192 exit 1
193
194 - name: Run integration test
195 working-directory: test/integration
196 run: |
197 export RSPAMD_HOST=localhost
198 export CONTROLLER_PORT=50002
199 export PROXY_PORT=50004
200 export PASSWORD=q1
201 export TEST_PROXY=true
202
203 # Verify corpus exists
204 if [ ! -d "data/corpus/spam" ] || [ ! -d "data/corpus/ham" ]; then
205 echo "ERROR: Corpus directories not found"
206 echo "Expected: data/corpus/spam and data/corpus/ham"
207 ls -la data/
208 ls -la data/corpus/ || true
209 exit 1
210 fi
211
212 echo "Corpus downloaded successfully:"
213 ls -lh data/corpus/
214
215 ./scripts/integration-test.sh
216
217 - name: Collect Docker logs
218 if: always()
219 working-directory: test/integration
220 run: |
221 echo "=== Collecting logs from all services ==="
222 docker compose logs --no-color > docker-compose-logs.txt 2>&1 || true
223 echo "=== Direct container logs ==="
224 docker logs rspamd-main >> docker-compose-logs.txt 2>&1 || true
225 docker logs rspamd-redis >> docker-compose-logs.txt 2>&1 || true
226
227 - name: Stop Docker Compose
228 if: always()
229 working-directory: test/integration
230 run: |
231 echo "Stopping containers to trigger ASAN log flush..."
232 docker compose down -v
233
234 - name: Check AddressSanitizer logs
235 if: always()
236 working-directory: test/integration
237 run: |
238 echo "=== Checking for ASAN logs after container shutdown ==="
239 ls -la data/
240
241 # Fix permissions on ASAN logs created by Docker
242 sudo chmod -R 644 data/asan.log* 2>/dev/null || true
243
244 # Check if ASAN logs exist
245 if ls data/asan.log* 1> /dev/null 2>&1; then
246 echo "✓ ASAN logs found"
247 ./scripts/check-asan-logs.sh || echo "Memory issues detected, but continuing..."
248 else
249 echo "⚠ No ASAN logs found after container shutdown"
250 fi
251
252 - name: Upload results
253 if: always()
254 uses: actions/upload-artifact@v4
255 with:
256 name: integration-test-results
257 path: |
258 test/integration/data/results.json
259 test/integration/data/proxy_results.json
260 test/integration/data/asan.log*
261 test/integration/data/*.log
262 retention-days: 7
263
264 - name: Upload Docker logs
265 if: always()
266 uses: actions/upload-artifact@v4
267 with:
268 name: docker-logs
269 path: |
270 test/integration/docker-compose-logs.txt
271 retention-days: 7
272 continue-on-error: true
273
274 - name: Test summary
275 if: always()
276 run: |
277 echo "## Integration Test Results" >> $GITHUB_STEP_SUMMARY
278 echo "" >> $GITHUB_STEP_SUMMARY
279
280 if [ -f "test/integration/data/results.json" ]; then
281 TOTAL=$(jq '. | length' test/integration/data/results.json)
282 echo "- Total emails scanned: $TOTAL" >> $GITHUB_STEP_SUMMARY
283 echo "- Results saved to artifacts" >> $GITHUB_STEP_SUMMARY
284 else
285 echo "- ❌ No results file generated" >> $GITHUB_STEP_SUMMARY
286 fi