| 1 |
#include "../source/utf8.h" |
| 2 |
#include <iostream> |
| 3 |
#include <fstream> |
| 4 |
#include <string> |
| 5 |
#include <vector> |
| 6 |
|
| 7 |
|
| 8 |
using namespace std; |
| 9 |
|
| 10 |
int main(int argc, char** argv) |
| 11 |
{ |
| 12 |
if (argc != 2) { |
| 13 |
cout << "\nUsage: docsample filename\n"; |
| 14 |
return 0; |
| 15 |
} |
| 16 |
const char* test_file_path = argv[1]; |
| 17 |
// Open the test file (must be UTF-8 encoded) |
| 18 |
ifstream fs8(test_file_path); |
| 19 |
if (!fs8.is_open()) { |
| 20 |
cout << "Could not open " << test_file_path << endl; |
| 21 |
return 0; |
| 22 |
} |
| 23 |
|
| 24 |
unsigned line_count = 1; |
| 25 |
string line; |
| 26 |
// Play with all the lines in the file |
| 27 |
while (getline(fs8, line)) { |
| 28 |
// check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function) |
| 29 |
#if __cplusplus >= 201103L // C++ 11 or later |
| 30 |
auto end_it = utf8::find_invalid(line.begin(), line.end()); |
| 31 |
#else |
| 32 |
string::iterator end_it = utf8::find_invalid(line.begin(), line.end()); |
| 33 |
#endif // C++ 11 |
| 34 |
if (end_it != line.end()) { |
| 35 |
cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n"; |
| 36 |
cout << "This part is fine: " << string(line.begin(), end_it) << "\n"; |
| 37 |
} |
| 38 |
// Get the line length (at least for the valid part) |
| 39 |
ptrdiff_t length = utf8::distance(line.begin(), end_it); |
| 40 |
cout << "Length of line " << line_count << " is " << length << "\n"; |
| 41 |
|
| 42 |
// Convert it to utf-16 |
| 43 |
#if __cplusplus >= 201103L // C++ 11 or later |
| 44 |
u16string utf16line = utf8::utf8to16(line); |
| 45 |
#else |
| 46 |
vector<unsigned short> utf16line; |
| 47 |
utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line)); |
| 48 |
#endif // C++ 11 |
| 49 |
// And back to utf-8; |
| 50 |
#if __cplusplus >= 201103L // C++ 11 or later |
| 51 |
string utf8line = utf8::utf16to8(utf16line); |
| 52 |
#else |
| 53 |
string utf8line; |
| 54 |
utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line)); |
| 55 |
#endif // C++ 11 |
| 56 |
// Confirm that the conversion went OK: |
| 57 |
if (utf8line != string(line.begin(), end_it)) |
| 58 |
cout << "Error in UTF-16 conversion at line: " << line_count << "\n"; |
| 59 |
|
| 60 |
line_count++; |
| 61 |
} |
| 62 |
|
| 63 |
return 0; |
| 64 |
} |