| 1 |
/* |
| 2 |
* guess.c - guessing character encoding |
| 3 |
* |
| 4 |
* Copyright (c) 2000-2024 Shiro Kawai <[email protected]> |
| 5 |
* |
| 6 |
* Redistribution and use in source and binary forms, with or without |
| 7 |
* modification, are permitted provided that the following conditions |
| 8 |
* are met: |
| 9 |
* |
| 10 |
* 1. Redistributions of source code must retain the above copyright |
| 11 |
* notice, this list of conditions and the following disclaimer. |
| 12 |
* |
| 13 |
* 2. Redistributions in binary form must reproduce the above copyright |
| 14 |
* notice, this list of conditions and the following disclaimer in the |
| 15 |
* documentation and/or other materials provided with the distribution. |
| 16 |
* |
| 17 |
* 3. Neither the name of the authors nor the names of its contributors |
| 18 |
* may be used to endorse or promote products derived from this |
| 19 |
* software without specific prior written permission. |
| 20 |
* |
| 21 |
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 22 |
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 23 |
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 24 |
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 25 |
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 26 |
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED |
| 27 |
* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| 28 |
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| 29 |
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| 30 |
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 31 |
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 32 |
*/ |
| 33 |
|
| 34 |
#include <gauche.h> |
| 35 |
#include <gauche/priv/configP.h> |
| 36 |
#include <gauche/extend.h> |
| 37 |
#include "charconv.h" |
| 38 |
|
| 39 |
typedef struct guess_arc_rec { |
| 40 |
unsigned int next; /* next state */ |
| 41 |
double score; /* score */ |
| 42 |
} guess_arc; |
| 43 |
|
| 44 |
typedef struct guess_dfa_rec { |
| 45 |
signed char (*states)[256]; |
| 46 |
guess_arc *arcs; |
| 47 |
int state; |
| 48 |
double score; |
| 49 |
} guess_dfa; |
| 50 |
|
| 51 |
#define DFA_INIT(st, ar) \ |
| 52 |
{ st, ar, 0, 1.0 } |
| 53 |
|
| 54 |
#define DFA_NEXT(dfa, ch) \ |
| 55 |
do { \ |
| 56 |
int arc__; \ |
| 57 |
if (dfa.state >= 0) { \ |
| 58 |
arc__ = dfa.states[dfa.state][ch]; \ |
| 59 |
if (arc__ < 0) { \ |
| 60 |
dfa.state = -1; \ |
| 61 |
} else { \ |
| 62 |
dfa.state = dfa.arcs[arc__].next; \ |
| 63 |
dfa.score *= dfa.arcs[arc__].score; \ |
| 64 |
} \ |
| 65 |
} \ |
| 66 |
} while (0) |
| 67 |
|
| 68 |
#define DFA_ALIVE(dfa) (dfa.state >= 0) |
| 69 |
|
| 70 |
/* include DFA table generated by guess.scm */ |
| 71 |
#include "guess_tab.c" |
| 72 |
|
| 73 |
static const char *guess_jp(const char *buf, ScmSize buflen, |
| 74 |
void *data SCM_UNUSED) |
| 75 |
{ |
| 76 |
guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar); |
| 77 |
guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar); |
| 78 |
guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar); |
| 79 |
|
| 80 |
for (ScmSize i=0; i<buflen; i++) { |
| 81 |
int c = (unsigned char)buf[i]; |
| 82 |
|
| 83 |
/* special treatment of jis escape sequence */ |
| 84 |
if (c == 0x1b) { |
| 85 |
if (i < buflen-1) { |
| 86 |
c = (unsigned char)buf[++i]; |
| 87 |
if (c == '$' || c == '(') return "ISO-2022-JP"; |
| 88 |
} |
| 89 |
} |
| 90 |
|
| 91 |
if (DFA_ALIVE(eucj)) { |
| 92 |
if (!DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) return "EUC-JP"; |
| 93 |
DFA_NEXT(eucj, c); |
| 94 |
} |
| 95 |
if (DFA_ALIVE(sjis)) { |
| 96 |
if (!DFA_ALIVE(eucj) && !DFA_ALIVE(utf8)) return "Shift_JIS"; |
| 97 |
DFA_NEXT(sjis, c); |
| 98 |
} |
| 99 |
if (DFA_ALIVE(utf8)) { |
| 100 |
if (!DFA_ALIVE(sjis) && !DFA_ALIVE(eucj)) return "UTF-8"; |
| 101 |
DFA_NEXT(utf8, c); |
| 102 |
} |
| 103 |
|
| 104 |
if (!DFA_ALIVE(eucj) && !DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) { |
| 105 |
/* we ran out the possibilities */ |
| 106 |
return NULL; |
| 107 |
} |
| 108 |
} |
| 109 |
|
| 110 |
/* Now, we have ambigous code. Pick the highest score. If more than |
| 111 |
one candidate tie, pick the default encoding. */ |
| 112 |
guess_dfa *top = NULL; |
| 113 |
if (DFA_ALIVE(eucj)) top = &eucj; |
| 114 |
if (DFA_ALIVE(utf8)) { |
| 115 |
if (top) { |
| 116 |
if (top->score <= utf8.score) top = &utf8; |
| 117 |
} else { |
| 118 |
top = &utf8; |
| 119 |
} |
| 120 |
} |
| 121 |
if (DFA_ALIVE(sjis)) { |
| 122 |
if (top) { |
| 123 |
if (top->score < sjis.score) top = &sjis; |
| 124 |
} else { |
| 125 |
top = &sjis; |
| 126 |
} |
| 127 |
} |
| 128 |
|
| 129 |
if (top == &eucj) return "EUC-JP"; |
| 130 |
if (top == &utf8) return "UTF-8"; |
| 131 |
if (top == &sjis) return "Shift_JIS"; |
| 132 |
return NULL; |
| 133 |
} |
| 134 |
|
| 135 |
|
| 136 |
/* |
| 137 |
* Initialization |
| 138 |
*/ |
| 139 |
|
| 140 |
void Scm_Init_convguess(void) |
| 141 |
{ |
| 142 |
Scm_RegisterCodeGuessingProc("*JP", guess_jp, NULL); |
| 143 |
} |