Debian Codegraph @

scripts/kbcompletion/eval.cpp

raw
1 /**
2 * Copyright (c) 2017-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the MIT license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 #include <unordered_map>
10 #include <iostream>
11 #include <fstream>
12 #include <string>
13 #include <vector>
14
15 std::string EOS = "</s>";
16
17 bool readWord(std::istream& in, std::string& word)
18 {
19 char c;
20 std::streambuf& sb = *in.rdbuf();
21 word.clear();
22 while ((c = sb.sbumpc()) != EOF) {
23 if (c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\v' ||
24 c == '\f' || c == '\0') {
25 if (word.empty()) {
26 if (c == '\n') {
27 word += EOS;
28 return true;
29 }
30 continue;
31 } else {
32 if (c == '\n')
33 sb.sungetc();
34 return true;
35 }
36 }
37 word.push_back(c);
38 }
39 in.get();
40 return !word.empty();
41 }
42
43 int main(int argc, char** argv) {
44 int k = 10;
45 if (argc < 4) {
46 std::cerr<<"eval <pred> <gt> <kb> [<k>]"<<std::endl;
47 exit(1);
48 }
49 if (argc == 5) { k = atoi(argv[4]);}
50
51 std::string predfn(argv[1]);
52 std::ifstream predf(predfn);
53 std::string gtfn(argv[2]);
54 std::ifstream gtf(gtfn);
55 std::string kbfn(argv[3]);
56 std::ifstream kbf(kbfn);
57
58 if (!predf.is_open() || !gtf.is_open() || !kbf.is_open()) {
59 std::cerr << "Files cannot be opened!" << std::endl;
60 exit(EXIT_FAILURE);
61 }
62
63 std::unordered_map< std::string,
64 std::unordered_map< std::string, bool > > KB;
65
66 while (kbf.peek() != EOF) {
67 std::string label, key, word;
68 while (readWord(kbf, word)) {
69 if (word == EOS) {break;}
70 if (word.find("__label__") == 0) {label = word;}
71 else {key += "|" + word;}
72 }
73 KB[key][label] = true;
74 }
75 kbf.close();
76
77 double precision = 0.0;
78 int32_t nexamples = 0;
79 while (predf.peek() != EOF || gtf.peek() != EOF) {
80 if (predf.peek() == EOF || gtf.peek() == EOF) {
81 std::cerr<<"pred / gt files have diff sizes"<<std::endl;
82 exit(1);
83 }
84 std::string label, key, word;
85
86 while (readWord(gtf, word)) {
87 if (word == EOS) {break;}
88 if ( word.find("__label__") == 0) {label = word;}
89 else {key += "|" + word;}
90 }
91 if (KB.find(key) == KB.end()) {
92 std::cerr<<"empty key!"<<std::endl; exit(1);
93 }
94
95 int count = 0;bool eval = true;
96 while (readWord(predf, word)) {
97 if (word == EOS) {break;}
98 if (!eval) {continue;}
99 if (label == word) {precision += 1.0; eval = false;}
100 else if (KB[key].find(word) == KB[key].end()) {count++;}
101 if (count == k) {eval = false;}
102 }
103 nexamples++;
104 }
105 predf.close(); gtf.close();
106 std::cout << "N:\t" << nexamples << std::endl;
107 std::cout << "R@" << k << "\t" << precision / nexamples << std::endl;
108 }