thdltools/Jskad
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fixed translation tool servlet issues: got rid of title, deleted white space, dealt with UTF8 better, etc.

Browse source
This commit is contained in:
amontano 2009-03-03 05:23:49 +00:00
parent 835e74c0cd
commit 5a0e454a2e
6 changed files with 1574 additions and 1434 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
source/org/thdl/tib/scanner/BasicTibetanTranscriptionConverter.java
View file

@ -46,6 +46,7 @@ public class BasicTibetanTranscriptionConverter implements FontConverterConstant
private static final int WYLIE_TO_ACIP=2;
private static final int UNICODE_TO_WYLIE=3;
private static final int WYLIE_TO_UNICODE=4;
private static final int TIBETAN_UNICODE_RANGE[] = {3840, 4095};
/** Converts from the Acip transliteration scheme to EWTS.*/
public static String acipToWylie(String acip)
@ -253,6 +254,18 @@ public class BasicTibetanTranscriptionConverter implements FontConverterConstant
return nuevaPalabra;*/
}
private static int getTibetanUnicodeStart(String unicode, int pos)
{
for(; pos < unicode.length(); pos++ ) if(unicode.codePointAt(pos)>=TIBETAN_UNICODE_RANGE[0] && unicode.codePointAt(pos)<=TIBETAN_UNICODE_RANGE[1]) return pos;
return -1;
}
private static int getTibetanUnicodeEnd(String unicode, int pos)
{
for(; pos < unicode.length(); pos++ ) if(unicode.codePointAt(pos)<TIBETAN_UNICODE_RANGE[0] || unicode.codePointAt(pos)>TIBETAN_UNICODE_RANGE[1]) return pos;
return pos;
}
/** Converts Tibetan Unicode to EWTS. */
public static String unicodeToWylie(String unicode)
{
@ -261,9 +274,9 @@ public class BasicTibetanTranscriptionConverter implements FontConverterConstant
TibetanDocument tibDoc;
StringBuffer errors;
int posStart=0, posEnd;
while((posStart = Manipulate.getTibetanUnicodeStart(unicode, posStart))>=0)
while((posStart = getTibetanUnicodeStart(unicode, posStart))>=0)
{
posEnd = Manipulate.getTibetanUnicodeEnd(unicode, posStart+1);
posEnd = getTibetanUnicodeEnd(unicode, posStart+1);
startString = unicode.substring(0, posStart);
tibetanString = unicode.substring(posStart, posEnd);
endString = unicode.substring(posEnd);

51
source/org/thdl/tib/scanner/Manipulate.java
View file

@ -14,21 +14,22 @@ created by Andres Montano Pellegrini are Copyright 2001 Andres Montano
Pellegrini. All Rights Reserved.
Contributor(s): ______________________________________.
*/
*/
package org.thdl.tib.scanner;
/** Miscelaneous static methods for the manipulation of Tibetan text.
@author Andr&eacute;s Montano Pellegrini
*/
*/
public class Manipulate
{
private static String endOfParagraphMarks = "/;|!:^@#$%=";
private static String endOfParagraphMarks = "/;|!:^@#$%=,";
private static String bracketMarks = "<>(){}[]";
private static String endOfSyllableMarks = " _\t";
private static String allStopMarkers = endOfSyllableMarks + endOfParagraphMarks + bracketMarks;
private static final int TIBETAN_UNICODE_RANGE[] = {3840, 4095};
private static String JSON_ESCAPABLES = "\"\\/";
/* public static String[] parseFields (String s, char delimiter)
{
@ -217,6 +218,7 @@ public class Manipulate
return ch>=0xF20 && ch<=0xF33;
}
public static boolean guessIfUnicode(String line)
{
char ch;
@ -256,14 +258,18 @@ public class Manipulate
int i=0;
char ch, ch2;
while (!isVowel(sil.charAt(i))) i++;
while (!isVowel(sil.charAt(i)))
{
i++;
if (i>=sil.length()) return null;
}
if (i==0) return "";
i--;
if (i==-1) return "";
if (sil.charAt(i)=='-') i--;
if (i>0 && sil.charAt(i)=='w') i--;
ch = sil.charAt(i);
// check to see if it is a subscript (y, r, l, w)
@ -271,7 +277,7 @@ public class Manipulate
{
switch (ch)
{
case 'r': case 'l': case 'w': i--;
case 'r': case 'l': i--;
break;
case 'y':
ch2 = sil.charAt(i-1);
@ -283,6 +289,7 @@ public class Manipulate
}
}
}
if (sil.charAt(i)=='+') i--;
if (i==0) return sil.substring(i,i+1);
ch = sil.charAt(i);
ch2 = sil.charAt(i-1);
@ -294,6 +301,8 @@ public class Manipulate
{
case 'k': case 'c': case 't': case 'p': case 'z':
return sil.substring(i-1,i+1);
case '+':
return sil.substring(i-2, i-1);
case 's':
if (i-2>=0 && sil.charAt(i-2)=='t') return "tsh";
else return "sh";
@ -429,6 +438,35 @@ public class Manipulate
return ncr.toString();
}
public static String toJSON(String str)
{
int pos, i, len;
for (i=0; i<str.length(); i++)
{
pos = JSON_ESCAPABLES.indexOf(str.charAt(i));
if (pos>=0)
{
len = str.length();
str = str.substring(0, i) + "\\" + str.substring(i, len);
i++;
}
}
str = replace(str, "\b", "\\b");
str = replace(str, "\f", "\\f");
str = replace(str, "\n", "\\n");
str = replace(str, "\r", "\\r");
str = replace(str, "\t", "\\t");
return str;
}
public static boolean containsLetters(String str)
{
int i=0;
if (str==null) return false;
while (i<str.length()) if (Character.isLetter(str.charAt(i++))) return true;
return false;
}
public static String unescape(String s) {
int i=0,len=s.length();
char c;
@ -459,5 +497,4 @@ public class Manipulate
for(; pos < unicode.length(); pos++ ) if(unicode.codePointAt(pos)<TIBETAN_UNICODE_RANGE[0] || unicode.codePointAt(pos)>TIBETAN_UNICODE_RANGE[1]) return pos;
return pos;
}
}

120
source/org/thdl/tib/scanner/OnLineScannerFilter.java
View file

@ -42,6 +42,7 @@ public class OnLineScannerFilter extends HttpServlet
private final static String dictNameProperty = "onlinescannerfilter.dict-file-name";
private final static String otherLinksProperty = "onlinescannerfilter.links-to-other-stuff";
private final static String moreLinksProperty = "onlinescannerfilter.links-to-more-stuff";
private final static String smallerLinksProperty = "onlinescannerfilter.links-to-smaller-stuff";
private final static String clearStr = "Clear";
private final static String buttonStr = "button";
private final static String scriptStr = "script";
@ -54,6 +55,7 @@ public class OnLineScannerFilter extends HttpServlet
public OnLineScannerFilter() //throws Exception
{
System.setProperty("java.awt.headless","true");
rb = ResourceBundle.getBundle(propertyFile);
sl = new ScannerLogger();
@ -75,7 +77,14 @@ public class OnLineScannerFilter extends HttpServlet
HttpServletResponse response) //throws IOException, ServletException
{
String answer, parrafo = null, checkboxName;
try
{
request.setCharacterEncoding("UTF8");
}
catch(Exception e)
{
// do nothing
}
// if this line is included in the constructor, it works on the orion server but not on wyllie!
ThdlOptions.setUserPreference("thdl.rely.on.system.tmw.fonts", true);
ThdlOptions.setUserPreference("thdl.rely.on.system.tm.fonts", true);
@ -102,15 +111,18 @@ public class OnLineScannerFilter extends HttpServlet
out.println("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">");
out.println("<html xmlns=\"http://www.w3.org/1999/xhtml\">");
out.println("<head>");
out.println(" <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />");
if (useTHDLBanner)
{
out.println(" <title>Tibetan and Himalayan Digital Library - The Online Tibetan to English Translation/Dictionary Tool</title>");
out.println(" <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />");
out.println(" <title>Tibetan and Himalayan Digital Library - The Online Tibetan to English Dictionary and Translation Tool</title>");
out.println(" <script type=\"text/javascript\" src=\"http://www.thdl.org/scripts/thdl_scripts.js\"></script>");
out.println(" <link rel=\"stylesheet\" type=\"text/css\" href=\"http://www.thdl.org/style/thdl-styles.css\"/>");
}
else
out.println(" <title>The Online Tibetan to English Translation/Dictionary Tool</title>");
{
out.println(" <title>The Online Tibetan to English Dictionary and Translation Tool</title>");
out.println(" <link rel=\"stylesheet\" type=\"text/css\" href=\"stylesheets/base.css\"/>");
}
out.println(" <meta name=\"keywords\" content=\"tibetan, english, dictionary, jim valby, rangjung yeshe, jeffrey hopkins, tsig mdzod chen mo, online, translation, scanner, parser, buddhism, language, processing, font, dharma, chos, tibet\">");
out.println(" <meta name=\"Description\" content=\"This Java tool takes Tibetan language passages and divides the passages up into their component phrases and words, and displays corresponding dictionary definitions.\">");
@ -158,9 +170,6 @@ public class OnLineScannerFilter extends HttpServlet
out.println("</div><!--END sub_banner-->");
out.println("<div id=\"main\">");
}
out.println("<h3 align=\"center\">The Online Tibetan to English Translation/Dictionary Tool</h3>");
try
{
out.println(rb.getString(otherLinksProperty));
@ -180,18 +189,18 @@ public class OnLineScannerFilter extends HttpServlet
}
out.println("<table border=\"0\" width=\"100%\">");
out.println(" <tr>");
out.println(" <td width=\"25%\">");
out.println(" <p>Display results in:</td>");
out.println(" <td width=\"75%\">");
out.println(" <p><input type=\"radio\" value=\"" + tibetanStr + "\" ");
out.println(" <td width=\"18%\" align=\"left\"><strong>Display results in:</strong></td>");
out.println(" <td width=\"41%\" align=\"right\">");
out.println(" <input type=\"radio\" value=\"" + tibetanStr + "\" ");
if (wantsTibetan) out.println("checked ");
out.println("name=\"" + scriptStr + "\">Tibetan script (using <a href=\"http://www.thdl.org/xml/show.php?xml=/tools/tibfonts.xml&l=uva10928423419921\" target=\"_blank\">Tibetan Machine Uni font</a>)<br/>");
out.println("name=\"" + scriptStr + "\">Tibetan script (<a href=\"http://www.thlib.org/tools/#wiki=/access/wiki/site/26a34146-33a6-48ce-001e-f16ce7908a6a/tibetan%20machine%20uni.html\" target=\"_top\">Tibetan Machine Uni</a> font)</td>");
out.println(" <td width=\"16%\" align=\"left\">");
out.println(" <input type=\"radio\" value=\"roman\" ");
if (!wantsTibetan) out.println("checked ");
out.println("name=\"" + scriptStr + "\">Roman script</td>");
out.println(" <td width=\"25%\" align=\"right\">");
out.println("<a href=\"http://www.thlib.org/tools/#wiki=/access/wiki/site/c06fa8cf-c49c-4ebc-007f-482de5382105/tibetan%20translation%20tool.html\" target=\"_top\">Help & Offline Installation</a></td>");
out.println(" </tr>");
out.println("</table>");
if (dictionaries!=null)
{
int i;
@ -200,7 +209,7 @@ public class OnLineScannerFilter extends HttpServlet
checkedDicts = new boolean[dictionaries.length];
/* out.println(" <tr>");
out.println("<td width=\""+ percent +"%\">Search in dictionaries:</td>");*/
out.println("<p>Search in dictionaries: ");
out.println("<tr><td colspan=\"4\"><strong>Search in dictionaries: </strong>");
allUnchecked=true;
for (i=0; i<dictionaries.length; i++)
{
@ -239,22 +248,43 @@ public class OnLineScannerFilter extends HttpServlet
out.print(">" + DictionarySource.defTags[i] + "&nbsp;&nbsp;&nbsp;");
// out.println(" + "</td>");
}
// out.println(" </tr>");
out.println(" </td></tr>");
}
// fix for updates
else ds = BitDictionarySource.getAllDictionaries();
// out.println("</table>");
out.println("</p>");
out.println("<table border=\"0\" width=\"100%\">");
// out.println("</p>");
// out.println("<table border=\"0\" width=\"100%\">");
out.println(" <tr>");
out.println(" <td width=\"35%\">");
out.println(" <p><strong>Input text:</strong></p>");
out.println(" </td>");
out.println(" <td width=\"65%\">");
out.println(" <p> <input type=\"submit\" name=\"" + buttonStr + "\" value=\"Translate\"> <input type=\"submit\" name=\"" + buttonStr + "\" value=\"" + clearStr + "\"></p>");
out.println(" </td>");
out.println(" <td><strong>Input text:</strong></td>");
out.println(" <td><input type=\"submit\" name=\"" + buttonStr + "\" value=\"Translate\"> <input type=\"submit\" name=\"" + buttonStr + "\" value=\"" + clearStr + "\"></td>");
out.println(" <td colspan\"2\">&nbsp;</td");
out.println(" </tr>");
out.println("</table>");
answer = request.getParameter(buttonStr);
String smallerLinks=null;
if (answer == null || answer != null && !answer.equals(clearStr))
{
parrafo = request.getParameter("parrafo");
}
if (parrafo==null)
{
try
{
smallerLinks = rb.getString(smallerLinksProperty);
}
catch (MissingResourceException e)
{
// do nothing
}
}
if (smallerLinks!=null)
{
out.println("<table width=\"100%\">");
out.println("<tr>");
out.println("<td>");
}
out.print("<textarea rows=\"5\" name=\"parrafo\" cols=\"40\"");
if (wantsTibetan) out.print(" class=\"tib\"");
@ -262,29 +292,31 @@ public class OnLineScannerFilter extends HttpServlet
// Paragraph should be empty if the user just clicked the clear button
answer = request.getParameter(buttonStr);
if (answer == null || answer != null && !answer.equals(clearStr))
if (parrafo!=null)
{
parrafo = request.getParameter("parrafo");
if (parrafo!=null) out.print(parrafo);
out.print(parrafo);
}
out.println("</textarea>");
if (smallerLinks!=null)
{
out.println("</td>");
out.println("<td>");
out.println(smallerLinks);
out.println("</td>");
out.println("</tr>");
out.println("</table>");
}
out.println("</textarea>");
out.println("</form>");
try
{
out.println(rb.getString(moreLinksProperty));
}
catch (MissingResourceException e)
{
// do nothing
}
if (parrafo != null)
{
sl.writeLog("4\t1");
if (ds!=null && !ds.isEmpty())
{
desglosar(parrafo, out, wantsTibetan);
}
}
else sl.writeLog("3\t1");
out.println(TibetanScanner.copyrightHTML);
@ -304,7 +336,7 @@ public class OnLineScannerFilter extends HttpServlet
{
//boolean hayMasLineas=true;
//int init = 0, fin;
//String linea;
String tmp;
Object words[];
if (!in.equals(""))
@ -331,6 +363,17 @@ public class OnLineScannerFilter extends HttpServlet
scanner.scanBody(in);
scanner.finishUp();
printText(pw, tibetan);
try
{
tmp = rb.getString(moreLinksProperty);
pw.println("<p>");
pw.println(tmp);
pw.println("</p>");
}
catch (MissingResourceException e)
{
// do nothing
}
printAllDefs(pw, tibetan);
scanner.clearTokens();
}
@ -393,8 +436,7 @@ public class OnLineScannerFilter extends HttpServlet
words = scanner.getWordArray(false);
if (words == null)
return;
if (words == null) return;
pw.println("<table border=\"1\" width=\"100%\">");
for (j = 0; j < words.length; j++) {

86
source/org/thdl/tib/scanner/RemoteScannerFilter.java
View file

@ -14,7 +14,7 @@ created by Andres Montano Pellegrini are Copyright 2001 Andres Montano
Pellegrini. All Rights Reserved.
Contributor(s): ______________________________________.
*/
*/
package org.thdl.tib.scanner;
import java.io.BufferedReader;
@ -32,15 +32,18 @@ import javax.servlet.ServletResponse;
@author Andr&eacute;s Montano Pellegrini
@see RemoteTibetanScanner
*/
*/
public class RemoteScannerFilter extends GenericServlet
{
private TibetanScanner scanner;
private BitDictionarySource ds;
private ScannerLogger sl;
private static final int INTERNAL = 1;
private static final int JSON = 2;
public RemoteScannerFilter()
{
System.setProperty("java.awt.headless","true");
ResourceBundle rb = ResourceBundle.getBundle("dictionary");
sl = new ScannerLogger();
@ -53,6 +56,7 @@ public class RemoteScannerFilter extends GenericServlet
sl.writeLog("1\t2");
sl.writeException(e);
}
scanner.getDictionaryDescriptions();
ds = scanner.getDictionarySource();
sl.writeLog("Creation\t2");
}
@ -60,7 +64,28 @@ public class RemoteScannerFilter extends GenericServlet
public void service(ServletRequest req, ServletResponse res) //throws ServletException, IOException
{
BufferedReader br;
int format, i, j, k;
try
{
req.setCharacterEncoding("UTF8");
}
catch(Exception e)
{
// do nothing
}
String linea, dicts = req.getParameter("dicts"), dicDescrip[], jwf = req.getParameter("jwf"), tag;
Definitions defs;
ByteDictionarySource dict_source;
if (jwf!=null) format = JSON;
else format = INTERNAL;
switch (format)
{
case INTERNAL:
res.setContentType ("text/plain");
break;
case JSON:
res.setContentType ("text/x-json");
}
sl.setUserIP(req.getRemoteAddr());
Word word = null, words[] = null;
@ -77,8 +102,6 @@ public class RemoteScannerFilter extends GenericServlet
return;
}
int i;
String linea, dicts = req.getParameter("dicts"), dicDescrip[];
if (dicts!=null)
{
@ -105,31 +128,29 @@ public class RemoteScannerFilter extends GenericServlet
}
}
try
if (format==JSON)
{
br = new BufferedReader(new InputStreamReader(req.getInputStream()));
out.println(jwf + "({\"words\":{");
}
catch (Exception e)
{
sl.writeLog("1\t2");
sl.writeException(e);
return;
}
/* FIXME: sometimes getDef raises a NullPointerException.
In the meantime, I'll just keep it from crashing
*/
sl.writeLog("4\t2");
try
{
scanner.clearTokens();
switch (format)
{
case INTERNAL:
br = req.getReader();
sl.writeLog("4\t2");
while((linea = br.readLine())!= null)
scanner.scanLine(linea);
br.close();
break;
case JSON:
linea = req.getParameter("text");
linea = Manipulate.NCR2UnicodeString(linea);
if (Manipulate.guessIfUnicode(linea)) linea = BasicTibetanTranscriptionConverter.unicodeToWylie(linea);
else if (Manipulate.guessIfAcip(linea)) linea = BasicTibetanTranscriptionConverter.acipToWylie(linea);
scanner.scanLine(linea);
}
scanner.finishUp();
words = scanner.getWordArray();
@ -137,10 +158,33 @@ public class RemoteScannerFilter extends GenericServlet
{
linea = words[i].getDef();
if (linea == null) continue;
switch (format)
{
case INTERNAL:
out.println(words[i].getWylie());
out.println(linea);
out.println();
break;
case JSON:
out.println("\"" + BasicTibetanTranscriptionConverter.wylieToHTMLUnicode(words[i].token) + "\": [");
defs = words[i].getDefs();
dict_source = (ByteDictionarySource)defs.getDictionarySource();
k=0;
for (j=0; j<defs.def.length; j++)
{
while (dict_source.isEmpty(k)) k++;
tag = dict_source.getTag(k);
k++;
out.println("\"" + tag + "\",");
out.print("\"" + Manipulate.toJSON(defs.def[j]) + "\"");
if (j==defs.def.length-1) out.println();
else out.println(",");
}
out.print("]");
if (i<words.length-1) out.println(",");
}
}
if (format==JSON) out.println("}});");
}
catch (Exception e)
{

11
source/org/thdl/tib/scanner/ScannerLogger.java
View file

@ -14,7 +14,7 @@ created by Andres Montano Pellegrini are Copyright 2001 Andres Montano
Pellegrini. All Rights Reserved.
Contributor(s): ______________________________________.
*/
*/
package org.thdl.tib.scanner;
import java.io.FileOutputStream;
@ -26,17 +26,22 @@ import java.util.ResourceBundle;
servlet version of the translation tool.
@author Andr&eacute;s Montano Pellegrini
*/
*/
public class ScannerLogger
{
private String fileName;
private String lastIP;
private boolean enabled;
public ScannerLogger()
{
String temp;
ResourceBundle rb = ResourceBundle.getBundle("dictionary");
fileName = rb.getString("remotescannerfilter.log-file-name");
temp = rb.getString("remotescannerfilter.logging-enabled");
if (temp==null) enabled = false;
else enabled = temp.toLowerCase().equals("yes");
lastIP = null;
}
@ -53,6 +58,7 @@ public class ScannerLogger
synchronized public void writeLog(String s)
{
if (!enabled) return;
PrintStream pw = getPrintStream();
if (lastIP!=null) pw.print(lastIP);
else pw.print("-");
@ -78,6 +84,7 @@ public class ScannerLogger
synchronized public void writeException(Exception e)
{
if (!enabled) return;
PrintStream pw = getPrintStream();
e.printStackTrace(pw);
pw.flush();

13
source/org/thdl/tib/scanner/TibetanScanner.java
View file

@ -14,7 +14,7 @@ created by Andres Montano Pellegrini are Copyright 2001 Andres Montano
Pellegrini. All Rights Reserved.
Contributor(s): ______________________________________.
*/
*/
package org.thdl.tib.scanner;
import org.thdl.util.SimplifiedLinkedList;
@ -24,13 +24,13 @@ import org.thdl.util.ThdlVersion;
/** Defines the core methods required to provide access to a dictionary; local or remote.
@author Andr&eacute;s Montano Pellegrini
*/
*/
public abstract class TibetanScanner
{
public static final String version = "The Tibetan to English Translation Tool, version 3.3.0 compiled on " + ThdlVersion.getTimeOfCompilation() + ". ";
public static final String copyrightUnicode="Copyright " + '\u00A9' + " 2000-200??6 by Andr" + '\u00E9' + "s Montano Pellegrini, all rights reserved.";
public static final String copyrightASCII="Copyright 2000-2006 by Andres Montano Pellegrini, all rights reserved.";
public static final String copyrightHTML="<hr><small><strong>" + version + "Copyright &copy; 2000-2006 by <a href=\"http://www.people.virginia.edu/~am2zb/\" target=\"_blank\">Andr&eacute;s Montano Pellegrini.</a><br/>All rights reserved.</strong></small>";
public static final String copyrightUnicode="Copyright " + '\u00A9' + " 2000-2009 by Andr" + '\u00E9' + "s Montano Pellegrini, all rights reserved.";
public static final String copyrightASCII="Copyright 2000-2009 by Andres Montano Pellegrini, all rights reserved.";
public static final String copyrightHTML="<hr><small><strong>" + version + "Copyright &copy; 2000-2009 by <a href=\"http://www.gaugeus.com/ramblings\" target=\"_blank\">Andr&eacute;s Montano Pellegrini.</a> All rights reserved.</strong></small>";
public static final int NORMAL_MODE=1;
public static final int DEBUG_MODE=2;
@ -242,9 +242,6 @@ public abstract class TibetanScanner
array[--n] = (Word) li.next();
}
}
return array;
}