| 1 |
/** |
| 2 |
* Copyright (c) 2017-present, Facebook, Inc. |
| 3 |
* All rights reserved. |
| 4 |
* |
| 5 |
* This source code is licensed under the MIT license found in the |
| 6 |
* LICENSE file in the root directory of this source tree. |
| 7 |
*/ |
| 8 |
|
| 9 |
#include <unordered_map> |
| 10 |
#include <iostream> |
| 11 |
#include <fstream> |
| 12 |
#include <string> |
| 13 |
#include <vector> |
| 14 |
|
| 15 |
std::string EOS = "</s>"; |
| 16 |
|
| 17 |
bool readWord(std::istream& in, std::string& word) |
| 18 |
{ |
| 19 |
char c; |
| 20 |
std::streambuf& sb = *in.rdbuf(); |
| 21 |
word.clear(); |
| 22 |
while ((c = sb.sbumpc()) != EOF) { |
| 23 |
if (c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\v' || |
| 24 |
c == '\f' || c == '\0') { |
| 25 |
if (word.empty()) { |
| 26 |
if (c == '\n') { |
| 27 |
word += EOS; |
| 28 |
return true; |
| 29 |
} |
| 30 |
continue; |
| 31 |
} else { |
| 32 |
if (c == '\n') |
| 33 |
sb.sungetc(); |
| 34 |
return true; |
| 35 |
} |
| 36 |
} |
| 37 |
word.push_back(c); |
| 38 |
} |
| 39 |
in.get(); |
| 40 |
return !word.empty(); |
| 41 |
} |
| 42 |
|
| 43 |
int main(int argc, char** argv) { |
| 44 |
int k = 10; |
| 45 |
if (argc < 4) { |
| 46 |
std::cerr<<"eval <pred> <gt> <kb> [<k>]"<<std::endl; |
| 47 |
exit(1); |
| 48 |
} |
| 49 |
if (argc == 5) { k = atoi(argv[4]);} |
| 50 |
|
| 51 |
std::string predfn(argv[1]); |
| 52 |
std::ifstream predf(predfn); |
| 53 |
std::string gtfn(argv[2]); |
| 54 |
std::ifstream gtf(gtfn); |
| 55 |
std::string kbfn(argv[3]); |
| 56 |
std::ifstream kbf(kbfn); |
| 57 |
|
| 58 |
if (!predf.is_open() || !gtf.is_open() || !kbf.is_open()) { |
| 59 |
std::cerr << "Files cannot be opened!" << std::endl; |
| 60 |
exit(EXIT_FAILURE); |
| 61 |
} |
| 62 |
|
| 63 |
std::unordered_map< std::string, |
| 64 |
std::unordered_map< std::string, bool > > KB; |
| 65 |
|
| 66 |
while (kbf.peek() != EOF) { |
| 67 |
std::string label, key, word; |
| 68 |
while (readWord(kbf, word)) { |
| 69 |
if (word == EOS) {break;} |
| 70 |
if (word.find("__label__") == 0) {label = word;} |
| 71 |
else {key += "|" + word;} |
| 72 |
} |
| 73 |
KB[key][label] = true; |
| 74 |
} |
| 75 |
kbf.close(); |
| 76 |
|
| 77 |
double precision = 0.0; |
| 78 |
int32_t nexamples = 0; |
| 79 |
while (predf.peek() != EOF || gtf.peek() != EOF) { |
| 80 |
if (predf.peek() == EOF || gtf.peek() == EOF) { |
| 81 |
std::cerr<<"pred / gt files have diff sizes"<<std::endl; |
| 82 |
exit(1); |
| 83 |
} |
| 84 |
std::string label, key, word; |
| 85 |
|
| 86 |
while (readWord(gtf, word)) { |
| 87 |
if (word == EOS) {break;} |
| 88 |
if ( word.find("__label__") == 0) {label = word;} |
| 89 |
else {key += "|" + word;} |
| 90 |
} |
| 91 |
if (KB.find(key) == KB.end()) { |
| 92 |
std::cerr<<"empty key!"<<std::endl; exit(1); |
| 93 |
} |
| 94 |
|
| 95 |
int count = 0;bool eval = true; |
| 96 |
while (readWord(predf, word)) { |
| 97 |
if (word == EOS) {break;} |
| 98 |
if (!eval) {continue;} |
| 99 |
if (label == word) {precision += 1.0; eval = false;} |
| 100 |
else if (KB[key].find(word) == KB[key].end()) {count++;} |
| 101 |
if (count == k) {eval = false;} |
| 102 |
} |
| 103 |
nexamples++; |
| 104 |
} |
| 105 |
predf.close(); gtf.close(); |
| 106 |
std::cout << "N:\t" << nexamples << std::endl; |
| 107 |
std::cout << "R@" << k << "\t" << precision / nexamples << std::endl; |
| 108 |
} |