Debian Codegraph @

tutorial/cpp/9-RefineComparison.cpp

raw
1 /*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8 #include <cassert>
9 #include <cstdio>
10 #include <cstdlib>
11 #include <random>
12
13 #include <faiss/IndexPQFastScan.h>
14 #include <faiss/IndexRefine.h>
15 #include <faiss/index_factory.h>
16 using idx_t = faiss::idx_t;
17
18 int main() {
19 int d = 64; // dimension
20 int nb = 100000; // database size
21 int nq = 10000; // nb of queries
22
23 std::mt19937 rng;
24 std::uniform_real_distribution<> distrib;
25
26 float* xb = new float[(int)(d * nb)];
27 float* xq = new float[(int)(d * nq)];
28
29 for (int i = 0; i < nb; i++) {
30 for (int j = 0; j < d; j++) {
31 xb[d * i + j] = distrib(rng);
32 }
33 xb[d * i] += i / 1000.;
34 }
35
36 for (int i = 0; i < nq; i++) {
37 for (int j = 0; j < d; j++) {
38 xq[d * i + j] = distrib(rng);
39 }
40 xq[d * i] += i / 1000.;
41 }
42
43 // Constructing the refine PQ index with SQfp16 with index factory
44 faiss::Index* index_fp16;
45 index_fp16 = faiss::index_factory(
46 d, "PQ32x4fs,Refine(SQfp16)", faiss::METRIC_L2);
47 index_fp16->train(nb, xb);
48 index_fp16->add(nb, xb);
49
50 // Constructing the refine PQ index with SQ8
51 faiss::Index* index_sq8;
52 index_sq8 =
53 faiss::index_factory(d, "PQ32x4fs,Refine(SQ8)", faiss::METRIC_L2);
54 index_sq8->train(nb, xb);
55 index_sq8->add(nb, xb);
56
57 int k = 10;
58 { // search xq
59 idx_t* I_fp16 = new idx_t[(int)(k * nq)];
60 float* D_fp16 = new float[(int)(k * nq)];
61 idx_t* I_sq8 = new idx_t[(int)(k * nq)];
62 float* D_sq8 = new float[(int)(k * nq)];
63
64 // Parameterization on k factor while doing search for index refinement
65 float k_factor = 3;
66 faiss::IndexRefineSearchParameters* params =
67 new faiss::IndexRefineSearchParameters();
68 params->k_factor = k_factor;
69
70 // Perform index search using different index refinement
71 index_fp16->search(nq, xq, k, D_fp16, I_fp16, params);
72 index_sq8->search(nq, xq, k, D_sq8, I_sq8, params);
73
74 printf("I_fp16=\n");
75 for (int i = nq - 5; i < nq; i++) {
76 for (int j = 0; j < k; j++) {
77 printf("%5zd ", I_fp16[i * k + j]);
78 }
79 printf("\n");
80 }
81
82 printf("I_sq8=\n");
83 for (int i = nq - 5; i < nq; i++) {
84 for (int j = 0; j < k; j++) {
85 printf("%5zd ", I_sq8[i * k + j]);
86 }
87 printf("\n");
88 }
89
90 delete[] I_fp16;
91 delete[] D_fp16;
92 delete[] I_sq8;
93 delete[] D_sq8;
94 delete params;
95
96 delete index_fp16;
97 delete index_sq8;
98 }
99
100 delete[] xb;
101 delete[] xq;
102
103 return 0;
104 }