| 1 |
#!/usr/bin/perl |
| 2 |
# By Jon Dehdari, 2006. Originally for ArabicLG, adapted for Shereen's thesis |
| 3 |
# This script started out as a small program that converts aramorph.pl output to |
| 4 |
# something that Arabic teachers will use. It has morphed into something |
| 5 |
# very different. That's my excuse that it's not pretty code, and I'm |
| 6 |
# sticking to it! |
| 7 |
|
| 8 |
use strict; |
| 9 |
use CGI qw(:standard); |
| 10 |
$CGI::POST_MAX=400000; |
| 11 |
my $cgi = new CGI; |
| 12 |
my %input; |
| 13 |
my $counter = 0; |
| 14 |
my ($newword, $romanized_word, $gloss, $nonhtml, $out); |
| 15 |
|
| 16 |
my $appended_file = $cgi->param ("appended_file"); |
| 17 |
my $preparsed_file = $cgi->param ("preparsed_file"); |
| 18 |
for my $key ( $cgi->param() ) { |
| 19 |
$input{$key} = $cgi->param($key); |
| 20 |
} |
| 21 |
|
| 22 |
if ($input{'save2file'} || $input{'append2file'} || $input{'save2xls'}) { $nonhtml = 1; } |
| 23 |
|
| 24 |
unless ($nonhtml) { |
| 25 |
print qq{Content-type: text/html; charset=windows-1256\n |
| 26 |
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//AR"> |
| 27 |
<html lang="ar"> |
| 28 |
<head><title>Arabic Text Analysis</title> |
| 29 |
<meta http-equiv="Content-Type" content="text/html; charset=windows-1256"> |
| 30 |
</head>\n<body style="background: url(arabesque1.png) repeat-x"><br/>\n |
| 31 |
<center><big><big>Arabic Text Analysis</big></big></center><br/><br/>\n |
| 32 |
<form action="buck2kirk.cgi" method="post" enctype="multipart/form-data"> |
| 33 |
}; |
| 34 |
$| = 1; # flushes out buffer |
| 35 |
#<style> |
| 36 |
# body \{ text-align\:right \} |
| 37 |
# textarea \{ text-align\:right \} |
| 38 |
#<\/style> |
| 39 |
} |
| 40 |
|
| 41 |
### prints all the cgi input, for development |
| 42 |
#for my $key ( keys %input ) { |
| 43 |
# print $key, ' = ', $input{$key}, "<br>\n"; |
| 44 |
#} |
| 45 |
|
| 46 |
$input{text_from} =~ s/\b(rm|mv|ln|passwd|shadow)\b//g; # scrub text input for security |
| 47 |
|
| 48 |
### I wanted to only call on aramorph_fast.pl once, since it's really slow to startup. As a result, I give all the user input lines to the morpho engine just once, then split it, then foreach it. |
| 49 |
|
| 50 |
my $scriptout; |
| 51 |
if ($input{text_from} || $preparsed_file) { # User has already input text into box or uploaded preparsed file |
| 52 |
if ($preparsed_file) { |
| 53 |
while (my $line = <$preparsed_file> ) { |
| 54 |
$scriptout .= $line ; |
| 55 |
# print "preparsed_file: " . <$preparsed_file> . " <br/>\n"; |
| 56 |
} |
| 57 |
} |
| 58 |
else { |
| 59 |
$scriptout = `echo -e '$input{text_from}' | nice ./aramorph_fast.pl -i $input{input_type} 2>/dev/null `; |
| 60 |
} |
| 61 |
|
| 62 |
|
| 63 |
my @scriptout = split /\n/, $scriptout; |
| 64 |
|
| 65 |
foreach $_ (@scriptout) { |
| 66 |
my ($romanized_lemma, $romanized_full, $pos_tag, $ar_word, $unknown_loop); |
| 67 |
|
| 68 |
if (m/SOLUTION \d+:/) { |
| 69 |
if (m/\* SOLUTION \d+:/) { # if starred word, erase checked status of first solution and cause checking of current solution by faking new word status |
| 70 |
$out =~ s/ checked //g; |
| 71 |
$newword = 1; |
| 72 |
} |
| 73 |
s/.*\[(.*?)\] (.*)/$1/g; |
| 74 |
$romanized_lemma = $1; |
| 75 |
$pos_tag = $2; |
| 76 |
$romanized_full = $2; |
| 77 |
$romanized_full =~ s/([^+ \/]+)\/[^+ ]+/$1/g; # removes part-of-speech info |
| 78 |
$romanized_full =~ tr/o//d; # remove sukuns |
| 79 |
} |
| 80 |
elsif (m/NEWLINE/) {} # Deal with this stuff later |
| 81 |
elsif (s/^ *\(GLOSS\):[+ ]+(.+?)[+ ]*$/$1/) { |
| 82 |
$gloss = $1; |
| 83 |
$gloss =~ s/(.*)<verb>$/$1/g; # removes final <verb> |
| 84 |
$out =~ s/(.*):: "/$1::${gloss}"/; |
| 85 |
$out .= " ${gloss} "; |
| 86 |
|
| 87 |
# print $out; |
| 88 |
# ($out, $gloss) = undef; |
| 89 |
# $| = 1; # flushes out buffer |
| 90 |
} |
| 91 |
elsif (m/Comment:\s+(\S+?)\s+NOT FOUND/) { |
| 92 |
my $templine = $1; |
| 93 |
$romanized_lemma = undef; |
| 94 |
$romanized_full = undef; |
| 95 |
$counter++; |
| 96 |
|
| 97 |
if ($input{output_type} eq "dt" ) { # DT |
| 98 |
if ($templine =~ m/\d+/) { # Is a number |
| 99 |
$out .= qq"<br/>\n |
| 100 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/NUMBER :: \" checked >" . buck2dt($romanized_word) . " - Number <br/> |
| 101 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/UNKNOWN :: \" >" . buck2dt($romanized_word) . " - Other <br/> |
| 102 |
"; |
| 103 |
} |
| 104 |
else { # Is not a number |
| 105 |
$out .= qq"<br/>\n |
| 106 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/NOUN_PROP :: \" checked >" . buck2dt($romanized_word) . " - Proper Noun <br/> |
| 107 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/NOUN :: \" >" . buck2dt($romanized_word) . " - Noun <br/> |
| 108 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/ADJ :: \" >" . buck2dt($romanized_word) . " - Adjective <br/> |
| 109 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/VERB_PERFECT :: \" >" . buck2dt($romanized_word) . " - Perfect Verb <br/> |
| 110 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/VERB_IMPERFECT :: \" >" . buck2dt($romanized_word) . " - Imperfect Verb <br/> |
| 111 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/UNKNOWN :: \" >" . buck2dt($romanized_word) . " - Other <br/> |
| 112 |
"; |
| 113 |
} |
| 114 |
} |
| 115 |
else { # Buckwalter transliteration |
| 116 |
if ($templine =~ m/\d+/) { # Is a number |
| 117 |
$out .= qq{<br/>\n |
| 118 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/NUMBER :: \" checked >$romanized_word - Number <br/> |
| 119 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/UNKNOWN :: \" >$romanized_word - Other <br/> |
| 120 |
}; |
| 121 |
} |
| 122 |
else { # Is not a number |
| 123 |
$out .= qq{<br/>\n |
| 124 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/NOUN_PROP :: \" checked >$romanized_word - Proper Noun <br/> |
| 125 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/NOUN :: \" >$romanized_word - Noun <br/> |
| 126 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/ADJ :: \" >$romanized_word - Adjective <br/> |
| 127 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/VERB_PERFECT :: \" >$romanized_word - Perfect Verb <br/> |
| 128 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/VERB_IMPERFECT :: \" >$romanized_word - Imperfect Verb <br/> |
| 129 |
<input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_word} :: ${romanized_word}/UNKNOWN :: \" >$romanized_word - Other <br/> |
| 130 |
}; |
| 131 |
} |
| 132 |
} |
| 133 |
|
| 134 |
} |
| 135 |
elsif (s/^INPUT STRING: (.*)/<br\/>\n$1 /) { |
| 136 |
print $out; |
| 137 |
($out, $gloss) = undef; |
| 138 |
$| = 1; # flushes out buffer |
| 139 |
|
| 140 |
$ar_word = $1; |
| 141 |
$out .= "<br/><br/>\n$ar_word"; |
| 142 |
$newword = 1; |
| 143 |
} |
| 144 |
elsif (s/^LOOK-UP WORD: (\S*)//) { $romanized_word = $1; } |
| 145 |
else { s/.*//g } |
| 146 |
|
| 147 |
my $romanized_lemma_html = $romanized_lemma; # protects "<" in html |
| 148 |
$romanized_lemma_html =~ s/_(\d+)//g; # eg. word_1 --> word |
| 149 |
|
| 150 |
s/\(null\)//g; |
| 151 |
tr/{/</; |
| 152 |
tr/\n/ /; |
| 153 |
s/\s+/ /g; # removes excessive spaces |
| 154 |
#s/[aiuo~]//g; # devocalize |
| 155 |
if (m/[a-zA-Z]/) { # some newline trickery |
| 156 |
if (m/NEWLINE/) { |
| 157 |
$out .= "<hr/><br/>\n"; |
| 158 |
} |
| 159 |
elsif ($romanized_lemma) { |
| 160 |
y/o//d; # remove sukuns |
| 161 |
if ($newword) { # auto select first radio button per word |
| 162 |
$counter++; |
| 163 |
### I'm holding off on compacting the following code until the dust settles a little bit |
| 164 |
if ($input{output_type} eq "dt" ) { |
| 165 |
$out .= "<br/>\n <input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_lemma} :: ${pos_tag} :: \" checked >" . buck2dt($romanized_lemma) . " " . buck2dt($romanized_full) . " "; |
| 166 |
} |
| 167 |
else { |
| 168 |
$romanized_lemma_html =~ s/<(?!br)/</g; # protects "<" in html |
| 169 |
$romanized_full =~ s/<(?!br)/</g; # protects "<" in html |
| 170 |
$out .= "<br/>\n <input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_lemma} :: ${pos_tag} :: \" checked >$romanized_lemma_html $romanized_full "; |
| 171 |
} |
| 172 |
$newword = 0; |
| 173 |
} |
| 174 |
else { # non-first parse |
| 175 |
if ($input{output_type} eq "dt" ) { |
| 176 |
$out .= "<br/>\n <input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_lemma} :: ${pos_tag} :: \">" . buck2dt($romanized_lemma) . " " . buck2dt($romanized_full) . " "; |
| 177 |
} |
| 178 |
else { |
| 179 |
$romanized_lemma_html =~ s/<(?!br)/</g; # protects "<" in html |
| 180 |
$romanized_full =~ s/<(?!br)/</g; # protects "<" in html |
| 181 |
$out .= "<br/>\n <input type=\"radio\" name=\"${counter}_${romanized_word}\" value=\"${romanized_lemma} :: ${pos_tag} :: \">$romanized_lemma_html $romanized_full "; |
| 182 |
} |
| 183 |
} |
| 184 |
} |
| 185 |
} |
| 186 |
} # foreach @out |
| 187 |
print <<EOF |
| 188 |
<hr/><br/> |
| 189 |
<input type="hidden" name="words_checked" value="true"> |
| 190 |
<br/> |
| 191 |
<input name="save2file" type="checkbox" value="true"> Save to New <b>Text</b> File <br/><br/> |
| 192 |
<input name="save2xls" type="checkbox" value="true"> Save to New <b>Excel</b> File <br/><br/> |
| 193 |
<input name="append2file" type="checkbox" value="true"> <b>Append</b> Existing Text File: <input name="appended_file" type="file" size="58"><br/><br/> |
| 194 |
EOF |
| 195 |
} # if $input{text_from} |
| 196 |
|
| 197 |
elsif ($input{words_checked}) { # User has already checked radio buttons of word senses |
| 198 |
undef $input{words_checked}; |
| 199 |
### prints all the cgi input, for development |
| 200 |
#print "<pre>"; |
| 201 |
if ($nonhtml ) { |
| 202 |
my $outfile; |
| 203 |
if ($input{append2file}) { |
| 204 |
# open (APPENDFILE, ">>$input{appended_file}") || die "cannot open: $!"; |
| 205 |
# while (my $line = <$input{appended_file}> or die "cannot open file") { |
| 206 |
# while (my $line = <$appended_file> or die "cannot open file") { |
| 207 |
while (my $line = <APPENDEDFILE>) { |
| 208 |
# print "line:$line\n"; |
| 209 |
$outfile .= $line; |
| 210 |
} |
| 211 |
} |
| 212 |
for my $key ( sort { $a <=> $b } keys %input ) { |
| 213 |
# if ($input{$key}) { print APPENDFILE "$key\t$input{$key}\n"; } |
| 214 |
if ($input{$key} && $key =~ /^\d+_/) { |
| 215 |
$outfile .= "${key} :: ${input{$key}}\n"; |
| 216 |
} |
| 217 |
} |
| 218 |
# print "\nSaved<br/>\n"; |
| 219 |
if ($input{'save2xls'}) {print "Content-type: application/vnd.ms-excel;\n\n"; |
| 220 |
undef $input{'save2xls'}; |
| 221 |
$outfile =~ s/\b(\d+)_/$1\t/g; |
| 222 |
$outfile =~ s/::/\t/g; |
| 223 |
print $outfile; |
| 224 |
} |
| 225 |
else {print "Content-type: text/plain;\n\n"; |
| 226 |
print <$appended_file> if $input{'append2file'};#temp |
| 227 |
print "\n$outfile\n" ; |
| 228 |
} |
| 229 |
# open APPENDEDFILE, "/tmp/$appended_file"; |
| 230 |
# close APPENDEDFILE; |
| 231 |
# close APPENDFILE; |
| 232 |
|
| 233 |
} |
| 234 |
else { |
| 235 |
for my $key ( sort { $a <=> $b } keys %input ) { |
| 236 |
if ($input{$key}) { print $key, ' = ', $input{$key}, "<br>\n"; } |
| 237 |
} |
| 238 |
} |
| 239 |
#print "</pre>"; |
| 240 |
} |
| 241 |
|
| 242 |
|
| 243 |
else { # No input from user, yet |
| 244 |
print <<EOF; |
| 245 |
Input Format: |
| 246 |
<select name="input_type"> |
| 247 |
<option value="dt">Dil Transliteration</option> |
| 248 |
<option value="roman">Buckwalter Transliteration</option> |
| 249 |
<option value="utf8">UTF-8</option> |
| 250 |
<!-- <option value="0">Unicode Decimal</option> --> |
| 251 |
<option value="cp1256" selected="selected">Windows 1256</option> |
| 252 |
<!-- <option value="7">ISO 8859-6</option> --> |
| 253 |
<!-- <option value="8">ArabTeX</option> --> |
| 254 |
</select><br/><br/> |
| 255 |
Output Format: |
| 256 |
<select name="output_type"> |
| 257 |
<option value="dt" selected="selected">Dil Transliteration</option> |
| 258 |
<option value="roman">Buckwalter Transliteration</option> |
| 259 |
<!-- <option value="utf8">UTF-8</option> --> |
| 260 |
<!-- <option value="0">Unicode Decimal</option> --> |
| 261 |
<!-- <option value="cp1256">Windows 1256</option> --> |
| 262 |
<!-- <option value="7">ISO 8859-6</option> --> |
| 263 |
<!-- <option value="8">ArabTeX</option> --> |
| 264 |
</select><br/><br/> |
| 265 |
Upload preparsed file: <input name="preparsed_file" type="file" size="29"> |
| 266 |
<br/><br/> |
| 267 |
<textarea style="text-align:right" name="text_from" rows="9" cols="70"></textarea> |
| 268 |
<br/><br/> |
| 269 |
EOF |
| 270 |
} |
| 271 |
|
| 272 |
unless ($nonhtml) { |
| 273 |
print '<input type="submit" value="Submit" />'."\n"; |
| 274 |
print "</body>\n</html>\n"; |
| 275 |
} |
| 276 |
|
| 277 |
# ============================================================== |
| 278 |
|
| 279 |
sub buck2dt { |
| 280 |
$_ = shift; |
| 281 |
|
| 282 |
tr/\'\|\>\&\<\}AbptvjHxd\*rzs\$SDTZEg_fqklmnhwYyFNKaui\~o\`\{/CMLWEYAbQtVjHxdvrzspSDTZcg_fqklmnhweyNUIaui\~o\`A/; # Buckwalter translit to DT |
| 283 |
|
| 284 |
return $_; |
| 285 |
|
| 286 |
} |
| 287 |
|
| 288 |
# ============================================================== |
| 289 |
|
| 290 |
sub unknown_guess { |
| 291 |
$_ = shift; |
| 292 |
|
| 293 |
# tr/\'\|\>\&\<\}AbptvjHxd\*rzs\$SDTZEg_fqklmnhwYyFNKaui\~o\`\{/CMLWEYAbQtVjHxdvrzspSDTZcg_fqklmnhweyNUIaui\~o\`A/; # Buckwalter translit to DT |
| 294 |
|
| 295 |
return $_; |
| 296 |
|
| 297 |
} |
| 298 |
|
| 299 |
# ============================================================== |