| 1 |
name: Integration Test |
| 2 |
|
| 3 |
permissions: |
| 4 |
contents: read |
| 5 |
|
| 6 |
on: |
| 7 |
schedule: |
| 8 |
# Run daily at 2 AM UTC |
| 9 |
- cron: '0 2 * * *' |
| 10 |
workflow_dispatch: |
| 11 |
inputs: |
| 12 |
corpus_url: |
| 13 |
description: 'URL to email corpus (zip file)' |
| 14 |
required: false |
| 15 |
default: 'https://github.com/rspamd/rspamd-test-corpus/releases/download/v1.0/rspamd-test-corpus.zip' |
| 16 |
|
| 17 |
jobs: |
| 18 |
integration-test: |
| 19 |
name: Integration & Load Test |
| 20 |
if: github.event_name != 'schedule' || github.repository == 'rspamd/rspamd' |
| 21 |
runs-on: ubuntu-latest |
| 22 |
timeout-minutes: 30 |
| 23 |
|
| 24 |
steps: |
| 25 |
- name: Checkout code |
| 26 |
uses: actions/checkout@v4 |
| 27 |
|
| 28 |
- name: Set up Python |
| 29 |
uses: actions/setup-python@v5 |
| 30 |
with: |
| 31 |
python-version: '3.11' |
| 32 |
|
| 33 |
- name: Install system dependencies |
| 34 |
run: | |
| 35 |
sudo apt-get update |
| 36 |
sudo apt-get install -y \ |
| 37 |
build-essential \ |
| 38 |
cmake \ |
| 39 |
ninja-build \ |
| 40 |
ragel \ |
| 41 |
libluajit-5.1-dev \ |
| 42 |
libglib2.0-dev \ |
| 43 |
libssl-dev \ |
| 44 |
libicu-dev \ |
| 45 |
libsodium-dev \ |
| 46 |
libhyperscan-dev \ |
| 47 |
libpcre2-dev \ |
| 48 |
libjemalloc-dev \ |
| 49 |
libunwind-dev \ |
| 50 |
libmagic-dev \ |
| 51 |
libarchive-dev \ |
| 52 |
libzstd-dev \ |
| 53 |
libbrotli-dev \ |
| 54 |
libfann-dev \ |
| 55 |
libstemmer-dev \ |
| 56 |
liblua5.1-dev \ |
| 57 |
redis-server \ |
| 58 |
sqlite3 \ |
| 59 |
libsqlite3-dev |
| 60 |
|
| 61 |
- name: Install Python dependencies |
| 62 |
run: | |
| 63 |
pip install requests |
| 64 |
|
| 65 |
- name: Build Rspamd |
| 66 |
working-directory: . |
| 67 |
run: | |
| 68 |
mkdir -p build install |
| 69 |
cd build |
| 70 |
cmake -DCMAKE_INSTALL_PREFIX=/usr \ |
| 71 |
-DCONFDIR=/etc/rspamd \ |
| 72 |
-DENABLE_COVERAGE=OFF \ |
| 73 |
-DENABLE_FULL_DEBUG=ON \ |
| 74 |
-DSANITIZE=address,leak \ |
| 75 |
-GNinja .. |
| 76 |
ninja |
| 77 |
DESTDIR="${GITHUB_WORKSPACE}/install" ninja install |
| 78 |
|
| 79 |
- name: Create static encryption keys |
| 80 |
working-directory: test/integration |
| 81 |
run: | |
| 82 |
# Use static keys to avoid LD_LIBRARY_PATH issues with rspamadm |
| 83 |
cat > .env.keys <<'EOF' |
| 84 |
# Rspamd integration test keys (static for testing) |
| 85 |
|
| 86 |
# Fuzzy worker keypair |
| 87 |
RSPAMD_FUZZY_WORKER_PRIVKEY=aergrxbdfppyo8x3upyw5brui1sug93ztyd9gcuuan141iy8zi1y |
| 88 |
RSPAMD_FUZZY_WORKER_PUBKEY=dsrea5fbqaqtzccno8roczeym5hgbokn5wxr5cyyhhzrqciiogpy |
| 89 |
|
| 90 |
# Fuzzy check encryption key (same as fuzzy worker pubkey) |
| 91 |
RSPAMD_FUZZY_ENCRYPTION_KEY=dsrea5fbqaqtzccno8roczeym5hgbokn5wxr5cyyhhzrqciiogpy |
| 92 |
|
| 93 |
# Normal worker keypair (for encrypted inter-worker communication) |
| 94 |
RSPAMD_WORKER_PRIVKEY=y8rq4a5rb7wj3xpefhx9t8gg3npfy4fswbwxyw7ma8hj46m36w1y |
| 95 |
RSPAMD_WORKER_PUBKEY=bnw1ga6h7q155qei7tzz9k5z4rb9msbf8gwra4jbzsyj8chg5n3y |
| 96 |
|
| 97 |
# Proxy worker keypair |
| 98 |
RSPAMD_PROXY_PRIVKEY=ad9cz7mhb881ngsr3pncxydm9d6t917hyrysywixwpzntzhikruy |
| 99 |
RSPAMD_PROXY_PUBKEY=bjkakz1r7zps7z774yn11gb9det5rw3dxb7z7559p5twzucneucy |
| 100 |
EOF |
| 101 |
|
| 102 |
echo "Static keys configured for testing" |
| 103 |
|
| 104 |
- name: Download email corpus |
| 105 |
working-directory: test/integration |
| 106 |
run: | |
| 107 |
# Use provided URL or default corpus from rspamd-test-corpus |
| 108 |
CORPUS_URL="${{ github.event.inputs.corpus_url }}" |
| 109 |
if [ -z "$CORPUS_URL" ]; then |
| 110 |
# Default: use latest release from rspamd-test-corpus |
| 111 |
CORPUS_URL="https://github.com/rspamd/rspamd-test-corpus/releases/latest/download/rspamd-test-corpus.zip" |
| 112 |
fi |
| 113 |
|
| 114 |
echo "Downloading corpus from: $CORPUS_URL" |
| 115 |
|
| 116 |
# Create data directory for corpus |
| 117 |
mkdir -p data |
| 118 |
chmod 777 data |
| 119 |
|
| 120 |
curl -L "$CORPUS_URL" -o data/corpus.zip |
| 121 |
|
| 122 |
# Extract corpus |
| 123 |
unzip data/corpus.zip -d data/ |
| 124 |
|
| 125 |
# The archive contains a 'corpus' directory, so we should have data/corpus/ now |
| 126 |
ls -lh data/corpus/ |
| 127 |
|
| 128 |
- name: Start Docker Compose |
| 129 |
working-directory: test/integration |
| 130 |
run: | |
| 131 |
docker compose up -d |
| 132 |
|
| 133 |
# Wait for services to be ready |
| 134 |
# Rspamd takes ~30-40s to compile TLD database and load maps |
| 135 |
echo "Waiting for services to start..." |
| 136 |
sleep 20 |
| 137 |
|
| 138 |
# Check services |
| 139 |
echo "=== Docker Compose Services Status ===" |
| 140 |
docker compose ps |
| 141 |
|
| 142 |
echo "" |
| 143 |
echo "=== Redis Logs ===" |
| 144 |
docker compose logs redis |
| 145 |
|
| 146 |
echo "" |
| 147 |
echo "=== Rspamd Logs ===" |
| 148 |
docker compose logs rspamd |
| 149 |
|
| 150 |
- name: Wait for Rspamd to be ready |
| 151 |
working-directory: test/integration |
| 152 |
run: | |
| 153 |
echo "Waiting for Rspamd services to initialize..." |
| 154 |
|
| 155 |
# Wait for healthcheck to pass |
| 156 |
for i in {1..60}; do |
| 157 |
if curl -s http://localhost:50002/ping > /dev/null 2>&1; then |
| 158 |
echo "✓ Rspamd Controller is ready!" |
| 159 |
|
| 160 |
# Also check proxy |
| 161 |
if curl -s http://localhost:50004/ping > /dev/null 2>&1; then |
| 162 |
echo "✓ Rspamd Proxy is ready!" |
| 163 |
else |
| 164 |
echo "⚠ WARNING: Proxy not responding, but continuing..." |
| 165 |
fi |
| 166 |
|
| 167 |
# Show Rspamd stat |
| 168 |
echo "" |
| 169 |
echo "=== Rspamd Status ===" |
| 170 |
curl -s http://localhost:50002/stat 2>/dev/null | head -20 || echo "Cannot get stat" |
| 171 |
|
| 172 |
exit 0 |
| 173 |
fi |
| 174 |
echo "Waiting for Rspamd... (attempt $i/60)" |
| 175 |
sleep 3 |
| 176 |
done |
| 177 |
|
| 178 |
echo "❌ Rspamd failed to start within timeout!" |
| 179 |
echo "" |
| 180 |
echo "=== Docker Compose Status ===" |
| 181 |
docker compose ps |
| 182 |
echo "" |
| 183 |
echo "=== Full Rspamd logs ===" |
| 184 |
docker compose logs rspamd |
| 185 |
echo "" |
| 186 |
echo "=== Checking for ASAN logs in container ===" |
| 187 |
docker compose exec -T rspamd ls -la /data/ || true |
| 188 |
docker compose exec -T rspamd cat /data/asan.log* 2>/dev/null || echo "No ASAN logs found" |
| 189 |
echo "" |
| 190 |
echo "=== Container stderr/stdout ===" |
| 191 |
docker logs rspamd-main 2>&1 || true |
| 192 |
exit 1 |
| 193 |
|
| 194 |
- name: Run integration test |
| 195 |
working-directory: test/integration |
| 196 |
run: | |
| 197 |
export RSPAMD_HOST=localhost |
| 198 |
export CONTROLLER_PORT=50002 |
| 199 |
export PROXY_PORT=50004 |
| 200 |
export PASSWORD=q1 |
| 201 |
export TEST_PROXY=true |
| 202 |
|
| 203 |
# Verify corpus exists |
| 204 |
if [ ! -d "data/corpus/spam" ] || [ ! -d "data/corpus/ham" ]; then |
| 205 |
echo "ERROR: Corpus directories not found" |
| 206 |
echo "Expected: data/corpus/spam and data/corpus/ham" |
| 207 |
ls -la data/ |
| 208 |
ls -la data/corpus/ || true |
| 209 |
exit 1 |
| 210 |
fi |
| 211 |
|
| 212 |
echo "Corpus downloaded successfully:" |
| 213 |
ls -lh data/corpus/ |
| 214 |
|
| 215 |
./scripts/integration-test.sh |
| 216 |
|
| 217 |
- name: Collect Docker logs |
| 218 |
if: always() |
| 219 |
working-directory: test/integration |
| 220 |
run: | |
| 221 |
echo "=== Collecting logs from all services ===" |
| 222 |
docker compose logs --no-color > docker-compose-logs.txt 2>&1 || true |
| 223 |
echo "=== Direct container logs ===" |
| 224 |
docker logs rspamd-main >> docker-compose-logs.txt 2>&1 || true |
| 225 |
docker logs rspamd-redis >> docker-compose-logs.txt 2>&1 || true |
| 226 |
|
| 227 |
- name: Stop Docker Compose |
| 228 |
if: always() |
| 229 |
working-directory: test/integration |
| 230 |
run: | |
| 231 |
echo "Stopping containers to trigger ASAN log flush..." |
| 232 |
docker compose down -v |
| 233 |
|
| 234 |
- name: Check AddressSanitizer logs |
| 235 |
if: always() |
| 236 |
working-directory: test/integration |
| 237 |
run: | |
| 238 |
echo "=== Checking for ASAN logs after container shutdown ===" |
| 239 |
ls -la data/ |
| 240 |
|
| 241 |
# Fix permissions on ASAN logs created by Docker |
| 242 |
sudo chmod -R 644 data/asan.log* 2>/dev/null || true |
| 243 |
|
| 244 |
# Check if ASAN logs exist |
| 245 |
if ls data/asan.log* 1> /dev/null 2>&1; then |
| 246 |
echo "✓ ASAN logs found" |
| 247 |
./scripts/check-asan-logs.sh || echo "Memory issues detected, but continuing..." |
| 248 |
else |
| 249 |
echo "⚠ No ASAN logs found after container shutdown" |
| 250 |
fi |
| 251 |
|
| 252 |
- name: Upload results |
| 253 |
if: always() |
| 254 |
uses: actions/upload-artifact@v4 |
| 255 |
with: |
| 256 |
name: integration-test-results |
| 257 |
path: | |
| 258 |
test/integration/data/results.json |
| 259 |
test/integration/data/proxy_results.json |
| 260 |
test/integration/data/asan.log* |
| 261 |
test/integration/data/*.log |
| 262 |
retention-days: 7 |
| 263 |
|
| 264 |
- name: Upload Docker logs |
| 265 |
if: always() |
| 266 |
uses: actions/upload-artifact@v4 |
| 267 |
with: |
| 268 |
name: docker-logs |
| 269 |
path: | |
| 270 |
test/integration/docker-compose-logs.txt |
| 271 |
retention-days: 7 |
| 272 |
continue-on-error: true |
| 273 |
|
| 274 |
- name: Test summary |
| 275 |
if: always() |
| 276 |
run: | |
| 277 |
echo "## Integration Test Results" >> $GITHUB_STEP_SUMMARY |
| 278 |
echo "" >> $GITHUB_STEP_SUMMARY |
| 279 |
|
| 280 |
if [ -f "test/integration/data/results.json" ]; then |
| 281 |
TOTAL=$(jq '. | length' test/integration/data/results.json) |
| 282 |
echo "- Total emails scanned: $TOTAL" >> $GITHUB_STEP_SUMMARY |
| 283 |
echo "- Results saved to artifacts" >> $GITHUB_STEP_SUMMARY |
| 284 |
else |
| 285 |
echo "- ❌ No results file generated" >> $GITHUB_STEP_SUMMARY |
| 286 |
fi |