Debian Codegraph @

.github/workflows/index-io-backward-compatibility.yml

raw
1 name: Index serialization backward compatibility test
2
3 on:
4 workflow_call:
5
6 env:
7 OMP_NUM_THREADS: '10'
8 MKL_THREADING_LAYER: GNU
9
10 jobs:
11 # Scenario 1: CMake writes files, then Conda reads them
12 cmake-write-conda-read:
13 name: CMake Write -> Conda Read
14 runs-on: ubuntu-latest
15 env:
16 # run_id does not change if we re-run the job, so this is safe for
17 # the partway-failure-rerun scenario.
18 SHARED_DATA_DIR: /tmp/faiss-serialization-backward-compatibility-${{ github.run_id }}-cmake-write
19 steps:
20 - name: Checkout
21 uses: actions/checkout@v4
22
23 # Step 1: Build with CMake and write files
24 - name: Build with CMake
25 uses: ./.github/actions/build_cmake
26 with:
27 upload_artifacts: 'false'
28
29 - name: Create shared data directory
30 shell: bash
31 run: |
32 mkdir -p ${{ env.SHARED_DATA_DIR }}
33 chmod 777 ${{ env.SHARED_DATA_DIR }}
34
35 - name: Run CMake writer (write Faiss index and metadata)
36 shell: bash
37 run: |
38 eval "$(conda shell.bash hook)"
39 conda activate
40 $CONDA/bin/python tests/index_io_backward_compatibility/cmake_writer.py ${{ env.SHARED_DATA_DIR }}
41
42 - name: Verify files were written
43 shell: bash
44 run: |
45 echo "Files created by CMake build:"
46 ls -lh ${{ env.SHARED_DATA_DIR }}
47
48 # Step 2: Install conda faiss-cpu and read files
49 - name: Clean cmake-built packages
50 shell: bash
51 run: |
52 eval "$(conda shell.bash hook)"
53 conda activate
54 # Remove packages that conflict with faiss-cpu
55 conda remove -y numpy scipy pytest gflags swig cmake make mkl mkl-devel || true
56
57 - name: Install faiss-cpu from pytorch channel
58 shell: bash
59 run: |
60 eval "$(conda shell.bash hook)"
61 conda activate
62 conda list
63 conda install -y -c pytorch faiss-cpu=1.13.1
64 conda list
65
66 - name: Run Conda reader (read Faiss index and verify)
67 shell: bash
68 run: |
69 eval "$(conda shell.bash hook)"
70 conda activate
71 python tests/index_io_backward_compatibility/conda_reader.py ${{ env.SHARED_DATA_DIR }}
72
73 - name: Upload artifacts from cmake->conda test
74 if: always()
75 uses: actions/upload-artifact@v4
76 with:
77 name: cmake-write-conda-read-data
78 path: ${{ env.SHARED_DATA_DIR }}
79
80 # Scenario 2: Conda writes files, then CMake reads them
81 conda-write-cmake-read:
82 name: Conda Write -> CMake Read
83 runs-on: ubuntu-latest
84 env:
85 SHARED_DATA_DIR: /tmp/faiss-serialization-backward-compatibility-${{ github.run_id }}-conda-write
86 steps:
87 - name: Checkout
88 uses: actions/checkout@v4
89 with:
90 fetch-depth: 0
91 fetch-tags: true
92
93 # Step 1: Install conda faiss-cpu package and write files
94 - name: Setup miniconda
95 uses: conda-incubator/setup-miniconda@v3
96 with:
97 python-version: '3.11'
98 miniforge-version: latest
99
100 - name: Install faiss-cpu from pytorch channel
101 shell: bash
102 run: |
103 eval "$(conda shell.bash hook)"
104 conda activate
105 # Install pre-built faiss-cpu
106 conda install -y -c pytorch faiss-cpu=1.13.1
107 conda list
108
109 - name: Create shared data directory
110 shell: bash
111 run: |
112 mkdir -p ${{ env.SHARED_DATA_DIR }}
113 chmod 777 ${{ env.SHARED_DATA_DIR }}
114
115 - name: Run Conda writer (write Faiss index and metadata)
116 shell: bash
117 run: |
118 eval "$(conda shell.bash hook)"
119 conda activate
120 python tests/index_io_backward_compatibility/conda_writer.py ${{ env.SHARED_DATA_DIR }}
121
122 - name: Verify files were written
123 shell: bash
124 run: |
125 echo "Files created by Conda build:"
126 ls -lh ${{ env.SHARED_DATA_DIR }}
127
128 # Step 2: Rebuild with CMake and read files
129 - name: Clean conda artifacts
130 shell: bash
131 run: |
132 # Uninstall conda-built faiss to avoid conflicts
133 eval "$(conda shell.bash hook)"
134 conda activate
135 conda uninstall -y faiss-cpu || true
136
137 - name: Build with CMake
138 uses: ./.github/actions/build_cmake
139 with:
140 setup_conda: 'false'
141 upload_artifacts: 'false'
142
143 - name: Run CMake reader (read Faiss index and verify)
144 shell: bash
145 run: |
146 eval "$(conda shell.bash hook)"
147 conda activate
148 $CONDA/bin/python tests/index_io_backward_compatibility/cmake_reader.py ${{ env.SHARED_DATA_DIR }}
149
150 - name: Upload artifacts from conda->cmake test
151 if: always()
152 uses: actions/upload-artifact@v4
153 with:
154 name: conda-write-cmake-read-data
155 path: ${{ env.SHARED_DATA_DIR }}