Debian Codegraph @

data/0.12-dev/gen_pms/cleanup2.cpp

raw
1 #include <iostream>
2 #include <string>
3 #include <vector>
4 #include <fstream>
5 #include <strstream>
6 #include <algorithm>
7 using namespace std;
8
9 #include "setutils.h"
10
11 #define DEBUG_SUBSETS 0
12 #define DEBUG_SMALLMERGE 0
13
14
15 // SMALLMERGE_RATIO: the golden ratio used for smallmerge algorithm.
16 const double SMALLMERGE_RATIO = .75;
17
18 // PITHY_FILTER: remove all meanings with less than this many synonyms listed.
19 const unsigned int PITHY_FILTER = 10;
20
21
22 template<class T>
23 void printvec(const vector<T>& v, const char* label)
24 {
25 cout << label << " = { ";
26 copy(v.begin(), v.end(), ostream_iterator<T>(cout, ", "));
27 cout << "}\n";
28 }
29
30
31 void cleanup(vector< vector<string> >& classes, vector< vector<string> >& output)
32 {
33 output.reserve(50);
34
35 /* cleanup2 doesn't need this, we have this already.
36 // First, destroy any classes that only have two words. There
37 // tend to be extremely obscure.
38 for(int i = 0;i < classes.size();++i)
39 {
40 if (classes[i].size() == 2)
41 classes[i].clear();
42
43 sort(classes[i].begin(), classes[i].end());
44 }
45 */
46
47 for(int i = 0;i < classes.size();++i)
48 {
49 if (classes[i].empty())
50 continue;
51
52 for(int j = i+1;j < classes.size();++j)
53 {
54 if (classes[j].empty())
55 continue;
56
57 vector<string> left, right, common;
58 set_comparison(left, right, common,
59 classes[i].begin(), classes[i].end(),
60 classes[j].begin(), classes[j].end());
61
62
63 // Now we will delete proper subsets. There shouldn't
64 // be many of these.
65 if (left.empty())
66 {
67 #if DEBUG_SUBSETS
68 cout << i << " is a subset of " << j << endl;
69 printvec(classes[i], " lhs");
70 printvec(classes[j], " rhs");
71 #endif // DEBUG_SUBSETS
72 classes[i].clear();
73 break;
74 }
75 else if (right.empty())
76 {
77 #if DEBUG_SUBSETS
78 cout << j << " is a subset of " << i << endl;
79 printvec(classes[i], " lhs");
80 printvec(classes[j], " rhs");
81 #endif
82 classes[j].clear();
83 }
84
85
86 // Now we will attempt to merge small sets into large sets.
87 // There should (hopefully) be many of these merges.
88 //
89 // We will make the convention that merges will be placed
90 // into classes[i]. In other words, we want classes[i] to
91 // become our large set, and clear out classes[j].
92 double lratio = double(common.size()) / double(common.size() + left.size());
93 if (lratio >= SMALLMERGE_RATIO)
94 {
95 #if DEBUG_SMALLMERGE
96 cout << "smallmerge: " << i << " into " << j << " (ratio=" << lratio << ")" << endl;
97 printvec(classes[i], " lhs");
98 printvec(classes[j], " rhs");
99 #endif
100
101 for(int k = 0;k < classes[j].size();++k)
102 {
103 const string& word = (classes[j])[k];
104 if (!binary_search(classes[i].begin(), classes[i].end(), word))
105 {
106 classes[i].push_back(word);
107 sort(classes[i].begin(), classes[i].end());
108 }
109 }
110
111 classes[j].clear();
112
113 --i;
114 break;
115
116 #if DEBUG_SMALLMERGE
117 cout << "Post merge setup: " << endl;
118 printvec(classes[i], " lhs");
119 printvec(classes[j], " rhs");
120 #endif
121 }
122
123 double rratio = double(common.size()) / double(common.size() + right.size());
124 if (rratio >= SMALLMERGE_RATIO)
125 {
126 #if DEBUG_SMALLMERGE
127 cout << "smallmerge: " << j << " into " << i << " (ratio=" << rratio << ")" << endl;
128 printvec(classes[i], " lhs");
129 printvec(classes[j], " rhs");
130 #endif
131
132 for(int k = 0;k < classes[j].size();++k)
133 {
134 const string& word = (classes[j])[k];
135 if (!binary_search(classes[i].begin(), classes[i].end(), word))
136 {
137 classes[i].push_back(word);
138 sort(classes[i].begin(), classes[i].end());
139 }
140 }
141
142 classes[j].clear();
143
144 --i;
145 break;
146
147 #if DEBUG_SMALLMERGE
148 cout << "Post merge setup: " << endl;
149 printvec(classes[i], " lhs");
150 printvec(classes[j], " rhs");
151 #endif
152 }
153 }
154 }
155
156 for(int i = 0;i < classes.size();++i)
157 {
158 // if (classes[i].size() > PITHY_FILTER)
159 // {
160 // copy(classes[i].begin(), classes[i].end(),
161 // ostream_iterator<string>(cout, " "));
162 // cout << endl;
163 output.push_back(classes[i]);
164 // }
165 }
166 }
167