LISTSERV mailing list manager LISTSERV 16.5

Help for CODE4LIB Archives


CODE4LIB Archives

CODE4LIB Archives


CODE4LIB@LISTS.CLIR.ORG


View:

Message:

[

First

|

Previous

|

Next

|

Last

]

By Topic:

[

First

|

Previous

|

Next

|

Last

]

By Author:

[

First

|

Previous

|

Next

|

Last

]

Font:

Proportional Font

LISTSERV Archives

LISTSERV Archives

CODE4LIB Home

CODE4LIB Home

CODE4LIB  April 2012

CODE4LIB April 2012

Subject:

Re: Modern NACO Normalization (esp. in java?)

From:

"LeVan,Ralph" <[log in to unmask]>

Reply-To:

Code for Libraries <[log in to unmask]>

Date:

Wed, 11 Apr 2012 12:04:43 -0400

Content-Type:

text/plain

Parts/Attachments:

Parts/Attachments

text/plain (2623 lines)

I'm pretty sure attachments don't work on the list, so I'm just pasting
my NACO normalizer below.  Note that there are 2007 versions of the
normalize() method in there.  This is used for all the VIAF and
Identities indexing.

Ralph

/*
 * NacoNormalize.java
 *
 * Created on July 11, 2007, 10:52 AM
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */

package ORG.oclc.util;


/**
 *
 * @author levan
 */
public class NacoNormalize {
    static String
blankedPunctuation="!\"$%()*-./:;<=>?@\\^_`{|}~\u007f\u00ba\u00bf\u00d7\
u00f7\u01c0\u01c1\u01c2\u01c3\u207a\u207b\u207c\u207d\u207e\u208a\u208b\
u208c\u208d\u208e";
    static String notAngleBrackets
="!\"$%()*-./:;=?@\\^_`{|}~\u007f\u00ba\u00bf\u00d7\u00f7\u01c0\u01c1\u0
1c2\u01c3\u207a\u207b\u207c\u207d\u207e\u208a\u208b\u208c\u208d\u208e";
    static String keepChars="0123456789#&+<>\u266d";
    static public String nacoNormalize(String s) {
        Pointer ptr=new Pointer(s);
        nacoNormalize(ptr, false, false, true);
        return ptr.toStringVar();
    }

    static public String nacoNormalize(char[] chars, int offset, int
length, boolean keepCaps, boolean stripHTML, boolean keepFirstComma) {
        Pointer ptr=new Pointer(chars, offset, length);
        nacoNormalize(ptr, keepCaps, stripHTML, keepFirstComma);
        return ptr.toStringVar();
    }

    static public String nacoNormalize(String s, boolean keepCaps,
boolean stripHTML, boolean keepFirstComma) {
        Pointer ptr=new Pointer(s);
        nacoNormalize(ptr, keepCaps, stripHTML, keepFirstComma);
        return ptr.toStringVar();
    }

    static private void nacoNormalize(Pointer ptr) {
        nacoNormalize(ptr, false, false, false);
    }

    static private void nacoNormalize(Pointer ptr, boolean keepCaps,
boolean stripHTML, boolean keepFirstComma) {
        char c;
        for(int i=ptr.offset; i<ptr.offset+ptr.length; i++) {
            c=ptr.chars[i];
            if(c==',') // special rule for first comma
                if(keepFirstComma) {
                    keepFirstComma=false;
                    continue;
                }
                else
                    c=ptr.chars[i]=' '; // blank commas

            if((!stripHTML && blankedPunctuation.indexOf(c)>=0) ||
              (notAngleBrackets.indexOf(c)>=0) ||
              (c>='\u00a0' && c<='\u00b1') || // latin-1 punctuation
              (c>='\u00b5' && c<='\u00b8') || // more latin-1
punctuation
              (c>='\u2000' && c<='\u206f') || // general punctuation
              (c>='\u20A0' && c<='\u20Cf') || // currency symbols
              (c>='\u2100' && c<='\u214f'))   // Letterlike Symbols
                  c=ptr.chars[i]=' ';
            // trim leading whitespace
            if(i==ptr.offset && Character.isWhitespace(c)) {
                ptr.offset++;
                ptr.length--;
                continue;
            }
            if((c>='a' && c<='z') || keepChars.indexOf(c)>=0) { //
lowercase letters
                continue;
            }
            if(c>='A' && c<='Z') { // Uppercase letters
                if(!keepCaps)
                    c=ptr.chars[i]=(char)(c+' '); // shift to lower case
                continue;
            }
            if(Character.isWhitespace(c) &&
Character.isWhitespace(ptr.chars[i-1])) { // delete the extra WhiteSpace
                System.arraycopy(ptr.chars, i+1,
                  ptr.chars, i, ptr.length-(i-ptr.offset+1));
                ptr.length--;
                i--;
                continue;
            }
            if(Character.isWhitespace(c)) {
                c=ptr.chars[i]=' ';
                continue;
            }

            // Special characters for truncation.
            if(c=='\u0001' || // ZERO_OR_ONE_WILD ('?')
               c=='\u0002' || // ZERO_OR_MORE_WILD ('*')
               c=='\u0003') // ONE_ONLY_WILD ('#')
                continue;
            
            if((c>='\u00b2' && c<='\u0233') || (c>='\u1e00' &&
c<='\u1eff') ||
              (c>='\u2070' && c<='\u208e'))
            switch(c) {
                case '\u2070': // superscript zero
                case '\u2080': // subscript zero
                    c=ptr.chars[i]='0';
                case '\u00b9': // superscript one
                    c=ptr.chars[i]='1';
                    continue;
                case '\u00b2': // superscript two
                    c=ptr.chars[i]='2';
                    continue;
                case '\u00b3': // superscript three
                    c=ptr.chars[i]='3';
                    continue;
                case '\u2074': // superscript four
                case '\u2084': // subscript four
                    c=ptr.chars[i]='4';
                case '\u2075': // superscript five
                case '\u2085': // subscript five
                    c=ptr.chars[i]='5';
                case '\u2076': // superscript six
                case '\u2086': // subscript six
                    c=ptr.chars[i]='6';
                case '\u2077': // superscript seven
                case '\u2087': // subscript seven
                    c=ptr.chars[i]='7';
                case '\u2078': // superscript eight
                case '\u2088': // subscript eight
                    c=ptr.chars[i]='8';
                case '\u2079': // superscript nine
                case '\u2089': // subscript nine
                    c=ptr.chars[i]='9';
                case '\u00bc': // one quarter
                    replaceChar(ptr, i, "1/4");
                    nacoNormalize(ptr);
                    return;
                case '\u00bd': // one half
                    replaceChar(ptr, i, "1/2");
                    nacoNormalize(ptr);
                    return;
                case '\u00be': // three quarters
                    replaceChar(ptr, i, "3/4");
                    nacoNormalize(ptr);
                    return;
                case '\u00c0':
                case '\u00c1':
                case '\u00c2':
                case '\u00c3':
                case '\u00c4':
                case '\u00c5':
                case '\u00e0':
                case '\u00e1':
                case '\u00e2':
                case '\u00e3':
                case '\u00e4':
                case '\u00e5':
                case '\u0100':
                case '\u0101':
                case '\u0102':
                case '\u0103':
                case '\u0104':
                case '\u0105':
                case '\u01cd':
                case '\u01ce':
                case '\u01de':
                case '\u01df':
                case '\u01e0':
                case '\u01e1':
                case '\u01fa':
                case '\u01fb':
                case '\u0200':
                case '\u0201':
                case '\u0202':
                case '\u0203':
                case '\u0226':
                case '\u0227':
                case '\u1e00':
                case '\u1e01':
                case '\u1e9a':
                case '\u1ea0':
                case '\u1ea1':
                case '\u1ea2':
                case '\u1ea3':
                case '\u1ea4':
                case '\u1ea5':
                case '\u1ea6':
                case '\u1ea7':
                case '\u1ea8':
                case '\u1ea9':
                case '\u1eaa':
                case '\u1eab':
                case '\u1eac':
                case '\u1ead':
                case '\u1eae':
                case '\u1eaf':
                case '\u1eb0':
                case '\u1eb1':
                case '\u1eb2':
                case '\u1eb3':
                case '\u1eb4':
                case '\u1eb5':
                case '\u1eb6':
                case '\u1eb7':
                    c=ptr.chars[i]='a';
                    continue;
                case '\u00c6': // uppercase digraph ae
                case '\u00e6': // lowercase digraph ae
                case '\u01e2':
                case '\u01e3':
                case '\u01fc':
                case '\u01fd':
                    replaceChar(ptr, i, "ae");
                    nacoNormalize(ptr);
                    return;
                case '\u0180':
                case '\u0181':
                case '\u0182':
                case '\u0183':
                case '\u0184':
                case '\u0185':
                case '\u1e02':
                case '\u1e03':
                case '\u1e04':
                case '\u1e05':
                case '\u1e06':
                case '\u1e07':
                    c=ptr.chars[i]='b';
                    continue;
                case '\u00c7':
                case '\u00e7':
                case '\u0106':
                case '\u0107':
                case '\u0108':
                case '\u0109':
                case '\u010a':
                case '\u010b':
                case '\u010c':
                case '\u010d':
                case '\u0186':
                case '\u0187':
                case '\u0188':
                case '\u1e08':
                case '\u1e09':
                    c=ptr.chars[i]='c';
                    continue;
                case '\u00d0': // uppercase eth
                case '\u00f0': // lowercase eth
                case '\u010e':
                case '\u010f':
                case '\u0189':
                case '\u018a':
                case '\u018b':
                case '\u018c':
                case '\u018d':
                case '\u1e0a':
                case '\u1e0b':
                case '\u1e0c':
                case '\u1e0d':
                case '\u1e0e':
                case '\u1e0f':
                case '\u1e10':
                case '\u1e11':
                case '\u1e12':
                case '\u1e13':
                case '\u0110': // uppercase Latin D with stroke
                case '\u0111': // lowercase Latin D with stroke
                    c=ptr.chars[i]='d';
                    continue;
                case '\u01c4':
                case '\u01c5':
                case '\u01c6':
                case '\u01f1':
                case '\u01f2':
                case '\u01f3':
                    replaceChar(ptr, i, "dz");
                    nacoNormalize(ptr);
                    return;
                case '\u00c8':
                case '\u00c9':
                case '\u00ca':
                case '\u00cb':
                case '\u00e8':
                case '\u00e9':
                case '\u00ea':
                case '\u00eb':
                case '\u0112':
                case '\u0113':
                case '\u0114':
                case '\u0115':
                case '\u0116':
                case '\u0117':
                case '\u0118':
                case '\u0119':
                case '\u011a':
                case '\u011b':
                case '\u018e':
                case '\u018f':
                case '\u0190':
                case '\u01dd':
                case '\u0204':
                case '\u0205':
                case '\u0206':
                case '\u0207':
                case '\u0228':
                case '\u0229':
                case '\u1e14':
                case '\u1e15':
                case '\u1e16':
                case '\u1e17':
                case '\u1e18':
                case '\u1e19':
                case '\u1e1a':
                case '\u1e1b':
                case '\u1e1c':
                case '\u1e1d':
                case '\u1eb8':
                case '\u1eb9':
                case '\u1eba':
                case '\u1ebb':
                case '\u1ebc':
                case '\u1ebd':
                case '\u1ebe':
                case '\u1ebf':
                case '\u1ec0':
                case '\u1ec1':
                case '\u1ec2':
                case '\u1ec3':
                case '\u1ec4':
                case '\u1ec5':
                case '\u1ec6':
                case '\u1ec7':
                    c=ptr.chars[i]='e';
                    continue;
                case '\u0191':
                case '\u0192':
                case '\u1e1e':
                case '\u1e1f':
                case '\u1e9b':
                    c=ptr.chars[i]='f';
                    continue;
                case '\u011c':
                case '\u011d':
                case '\u011e':
                case '\u011f':
                case '\u0120':
                case '\u0121':
                case '\u0122':
                case '\u0123':
                case '\u0193':
                case '\u0194':
                case '\u01e4':
                case '\u01e5':
                case '\u01e6':
                case '\u01e7':
                case '\u01f4':
                case '\u01f5':
                case '\u1e20':
                case '\u1e21':
                    c=ptr.chars[i]='g';
                    continue;
                case '\u0124':
                case '\u0125':
                case '\u0126':
                case '\u0127':
                case '\u021e':
                case '\u021f':
                case '\u1e22':
                case '\u1e23':
                case '\u1e24':
                case '\u1e25':
                case '\u1e26':
                case '\u1e27':
                case '\u1e28':
                case '\u1e29':
                case '\u1e2a':
                case '\u1e2b':
                case '\u1e96':
                    c=ptr.chars[i]='h';
                    continue;
                case '\u0195': // lowercase digraph hv
                    replaceChar(ptr, i, "hv");
                    nacoNormalize(ptr);
                    return;
                case '\u00cc':
                case '\u00cd':
                case '\u00ce':
                case '\u00cf':
                case '\u00ec':
                case '\u00ed':
                case '\u00ee':
                case '\u00ef':
                case '\u0128':
                case '\u0129':
                case '\u012a':
                case '\u012b':
                case '\u012c':
                case '\u012d':
                case '\u012e':
                case '\u012f':
                case '\u0130':
                case '\u0131':
                case '\u0196':
                case '\u0197':
                case '\u01cf':
                case '\u01d0':
                case '\u0208':
                case '\u0209':
                case '\u020a':
                case '\u020b':
                case '\u1e2c':
                case '\u1e2d':
                case '\u1e2e':
                case '\u1e2f':
                case '\u1ec8':
                case '\u1ec9':
                case '\u1eca':
                case '\u1ecb':
                case '\u2071': // superscript small letter i
                    c=ptr.chars[i]='i';
                    continue;
                case '\u0132': // uppercase digraph ij
                case '\u0133': // lowercase digraph ij
                    replaceChar(ptr, i, "ij");
                    nacoNormalize(ptr);
                    return;
                case '\u0134':
                case '\u0135':
                    c=ptr.chars[i]='j';
                    continue;
                case '\u0136':
                case '\u0137':
                case '\u0138':
                case '\u0198':
                case '\u0199':
                case '\u01e8':
                case '\u01e9':
                case '\u1e30':
                case '\u1e31':
                case '\u1e32':
                case '\u1e33':
                case '\u1e34':
                case '\u1e35':
                    c=ptr.chars[i]='k';
                    continue;
                case '\u0139':
                case '\u013a':
                case '\u013b':
                case '\u013c':
                case '\u013d':
                case '\u013e':
                case '\u013f':
                case '\u0140':
                case '\u0141':
                case '\u0142':
                case '\u019a':
                case '\u019b':
                case '\u0234':
                case '\u1e36':
                case '\u1e37':
                case '\u1e38':
                case '\u1e39':
                case '\u1e3a':
                case '\u1e3b':
                case '\u1e3c':
                case '\u1e3d':
                    c=ptr.chars[i]='l';
                    continue;
                case '\u01c7':
                case '\u01c8':
                case '\u01c9':
                    replaceChar(ptr, i, "lj");
                    nacoNormalize(ptr);
                    return;
                case '\u019c':
                case '\u1e3e':
                case '\u1e3f':
                case '\u1e40':
                case '\u1e41':
                case '\u1e42':
                case '\u1e43':
                    c=ptr.chars[i]='m';
                    continue;
                case '\u00d1':
                case '\u00f1':
                case '\u0143':
                case '\u0144':
                case '\u0145':
                case '\u0146':
                case '\u0147':
                case '\u0148':
                case '\u0149':
                case '\u014a':
                case '\u014b':
                case '\u019d':
                case '\u019e':
                case '\u01f8':
                case '\u01f9':
                case '\u0235':
                case '\u1e44':
                case '\u1e45':
                case '\u1e46':
                case '\u1e47':
                case '\u1e48':
                case '\u1e49':
                case '\u1e4a':
                case '\u1e4b':
                case '\u207f': // superscript latin small letter n
                    c=ptr.chars[i]='n';
                    continue;
                case '\u01ca':
                case '\u01cb':
                case '\u01cc':
                    replaceChar(ptr, i, "nj");
                    nacoNormalize(ptr);
                    return;
                case '\u00d2':
                case '\u00d3':
                case '\u00d4':
                case '\u00d5':
                case '\u00d6':
                case '\u00d8':
                case '\u00f2':
                case '\u00f3':
                case '\u00f4':
                case '\u00f5':
                case '\u00f6':
                case '\u00f8':
                case '\u014c':
                case '\u014d':
                case '\u014e':
                case '\u014f':
                case '\u0150':
                case '\u0151':
                case '\u019f':
                case '\u01a0':
                case '\u01a1':
                case '\u01d1':
                case '\u01d2':
                case '\u01fe':
                case '\u01ff':
                case '\u020c':
                case '\u020d':
                case '\u020e':
                case '\u020f':
                case '\u022a':
                case '\u022b':
                case '\u022c':
                case '\u022d':
                case '\u022e':
                case '\u022f':
                case '\u0230':
                case '\u0231':
                case '\u1e4c':
                case '\u1e4d':
                case '\u1e4e':
                case '\u1e4f':
                case '\u1e50':
                case '\u1e51':
                case '\u1e52':
                case '\u1e53':
                case '\u1ecc':
                case '\u1ecd':
                case '\u1ece':
                case '\u1ecf':
                case '\u1ed0':
                case '\u1ed1':
                case '\u1ed2':
                case '\u1ed3':
                case '\u1ed4':
                case '\u1ed5':
                case '\u1ed6':
                case '\u1ed7':
                case '\u1ed8':
                case '\u1ed9':
                case '\u1eda':
                case '\u1edb':
                case '\u1edc':
                case '\u1edd':
                case '\u1ede':
                case '\u1edf':
                case '\u1ee0':
                case '\u1ee1':
                case '\u1ee2':
                case '\u1ee3':
                    c=ptr.chars[i]='o';
                    continue;
                case '\u0152': // uppercase digraph oe
                case '\u0153': // lowercase digraph oe
                    replaceChar(ptr, i, "oe");
                    nacoNormalize(ptr);
                    return;
                case '\u01a2': // uppercase digraph oi
                case '\u01a3': // lowercase digraph oi
                    replaceChar(ptr, i, "oi");
                    nacoNormalize(ptr);
                    return;
                case '\u0222': // uppercase digraph ou
                case '\u0223': // lowercase digraph ou
                    replaceChar(ptr, i, "ou");
                    nacoNormalize(ptr);
                    return;
                case '\u01a4':
                case '\u01a5':
                case '\u1e54':
                case '\u1e55':
                case '\u1e56':
                case '\u1e57':
                    c=ptr.chars[i]='p';
                    continue;
                case '\u01ea':
                case '\u01eb':
                case '\u01ec':
                case '\u01ed':
                    c=ptr.chars[i]='q';
                    continue;
                case '\u0154':
                case '\u0155':
                case '\u0156':
                case '\u0157':
                case '\u0158':
                case '\u0159':
                case '\u01a6':
                case '\u0210':
                case '\u0211':
                case '\u0212':
                case '\u0213':
                case '\u1e58':
                case '\u1e59':
                case '\u1e5a':
                case '\u1e5b':
                case '\u1e5c':
                case '\u1e5d':
                case '\u1e5e':
                case '\u1e5f':
                    c=ptr.chars[i]='r';
                    continue;
                case '\u015a':
                case '\u015b':
                case '\u015c':
                case '\u015d':
                case '\u015e':
                case '\u015f':
                case '\u0160':
                case '\u0161':
                case '\u017f':
                case '\u01a7':
                case '\u01a8':
                case '\u01a9':
                case '\u01aa':
                case '\u0218':
                case '\u0219':
                case '\u1e60':
                case '\u1e61':
                case '\u1e62':
                case '\u1e63':
                case '\u1e64':
                case '\u1e65':
                case '\u1e66':
                case '\u1e67':
                case '\u1e68':
                case '\u1e69':
                    c=ptr.chars[i]='s';
                    continue;
                case '\u00df': // esszett
                    replaceChar(ptr, i, "ss");
                    nacoNormalize(ptr);
                    return;
                case '\u0162':
                case '\u0163':
                case '\u0164':
                case '\u0165':
                case '\u0166':
                case '\u0167':
                case '\u01ab':
                case '\u01ac':
                case '\u01ad':
                case '\u01ae':
                case '\u021a':
                case '\u021b':
                case '\u0236':
                case '\u1e6a':
                case '\u1e6b':
                case '\u1e6c':
                case '\u1e6d':
                case '\u1e6e':
                case '\u1e6f':
                case '\u1e70':
                case '\u1e71':
                case '\u1e97':
                    c=ptr.chars[i]='t';
                    continue;
                case '\u00de': // uppercase thorn
                case '\u00fe': // lowercase thorn
                    replaceChar(ptr, i, "th");
                    nacoNormalize(ptr);
                    return;
                case '\u01be': // lowercase digraph ts
                    replaceChar(ptr, i, "ts");
                    nacoNormalize(ptr);
                    return;
                case '\u00d9':
                case '\u00da':
                case '\u00db':
                case '\u00dc':
                case '\u00f9':
                case '\u00fa':
                case '\u00fb':
                case '\u00fc':
                case '\u0168':
                case '\u0169':
                case '\u016a':
                case '\u016b':
                case '\u016c':
                case '\u016d':
                case '\u016e':
                case '\u016f':
                case '\u0170':
                case '\u0171':
                case '\u0172':
                case '\u0173':
                case '\u01af':
                case '\u01b0':
                case '\u01b1':
                case '\u01d3':
                case '\u01d4':
                case '\u01d5':
                case '\u01d6':
                case '\u01d7':
                case '\u01d8':
                case '\u01d9':
                case '\u01da':
                case '\u01db':
                case '\u01dc':
                case '\u0214':
                case '\u0215':
                case '\u0216':
                case '\u0217':
                case '\u1e72':
                case '\u1e73':
                case '\u1e74':
                case '\u1e75':
                case '\u1e76':
                case '\u1e77':
                case '\u1e78':
                case '\u1e79':
                case '\u1e7a':
                case '\u1e7b':
                case '\u1ee4':
                case '\u1ee5':
                case '\u1ee6':
                case '\u1ee7':
                case '\u1ee8':
                case '\u1ee9':
                case '\u1eea':
                case '\u1eeb':
                case '\u1eec':
                case '\u1eed':
                case '\u1eee':
                case '\u1eef':
                case '\u1ef0':
                case '\u1ef1':
                    c=ptr.chars[i]='u';
                    continue;
                case '\u01b2':
                case '\u1e7c':
                case '\u1e7d':
                case '\u1e7e':
                case '\u1e7f':
                    c=ptr.chars[i]='v';
                    continue;
                case '\u0174':
                case '\u0175':
                case '\u01bf':
                case '\u01f6':
                case '\u01f7':
                case '\u1e80':
                case '\u1e81':
                case '\u1e82':
                case '\u1e83':
                case '\u1e84':
                case '\u1e85':
                case '\u1e86':
                case '\u1e87':
                case '\u1e88':
                case '\u1e89':
                case '\u1e98':
                    c=ptr.chars[i]='w';
                    continue;
                case '\u1e8a':
                case '\u1e8b':
                case '\u1e8c':
                case '\u1e8d':
                    c=ptr.chars[i]='x';
                    continue;
                case '\u00dd':
                case '\u00fd':
                case '\u00ff':
                case '\u0176':
                case '\u0177':
                case '\u0178':
                case '\u01b3':
                case '\u01b4':
                case '\u0232':
                case '\u0233':
                case '\u1e8e':
                case '\u1e8f':
                case '\u1e99':
                case '\u1ef2':
                case '\u1ef3':
                case '\u1ef4':
                case '\u1ef5':
                case '\u1ef6':
                case '\u1ef7':
                case '\u1ef8':
                case '\u1ef9':
                    c=ptr.chars[i]='y';
                    continue;
                case '\u0179':
                case '\u017a':
                case '\u017b':
                case '\u017c':
                case '\u017d':
                case '\u017e':
                case '\u01b5':
                case '\u01b6':
                case '\u01b7':
                case '\u01b8':
                case '\u01b9':
                case '\u01ba':
                case '\u01bb':
                case '\u01ee':
                case '\u01ef':
                case '\u021c':
                case '\u021d':
                case '\u0224':
                case '\u0225':
                case '\u1e90':
                case '\u1e91':
                case '\u1e92':
                case '\u1e93':
                case '\u1e94':
                case '\u1e95':
                    c=ptr.chars[i]='z';
                    continue;
                case '\u01bc':
                case '\u01bd':
                    c=ptr.chars[i]='5';
                    continue;
            }

            // anything that makes it this far should be deleted
            System.arraycopy(ptr.chars, i+1,
              ptr.chars, i, ptr.length-(i-ptr.offset+1));
            ptr.length--;
            i--;
        }
        // trim trailing whitespace and comma
        while(ptr.length>0 &&
          (Character.isWhitespace(ptr.chars[ptr.offset+ptr.length-1]) ||
           ptr.chars[ptr.offset+ptr.length-1]==','))
            ptr.length--;
    }


    static public String nacoNormalize2007(char[] chars, int offset, int
length, boolean keepFirstComma) {
        Pointer ptr=new Pointer(chars, offset, length);
        nacoNormalize2007(ptr, keepFirstComma, true);
        return ptr.toStringVar();
    }

    static public String nacoNormalize2007(char[] chars, int offset, int
length, boolean keepFirstComma, boolean trimTrailingSpaces) {
        Pointer ptr=new Pointer(chars, offset, length);
        nacoNormalize2007(ptr, keepFirstComma, trimTrailingSpaces);
        return ptr.toStringVar();
    }

    static public String nacoNormalize2007(String s, boolean
keepFirstComma) {
        Pointer ptr=new Pointer(s);
        nacoNormalize2007(ptr, keepFirstComma, true);
        return ptr.toStringVar();
    }

    static public String nacoNormalize2007(String s, boolean
keepFirstComma, boolean trimTrailingSpaces) {
        Pointer ptr=new Pointer(s);
        nacoNormalize2007(ptr, keepFirstComma, trimTrailingSpaces);
        return ptr.toStringVar();
    }

    static private void nacoNormalize2007(Pointer ptr, boolean
keepFirstComma, boolean trimTrailingSpaces) {
        boolean foundFirstComma=false, foundStartOfString=false;
        char c, c2;
        int type;
        for(int i=ptr.offset; i<ptr.offset+ptr.length; i++) {
            c=ptr.chars[i];

            // remove \u0098, \u009c pairs and everything between them
            if(c=='\u0098' || foundStartOfString) {
                foundStartOfString=true;
                // remove
                System.arraycopy(ptr.chars, i+1,
                  ptr.chars, i, ptr.length-(i-ptr.offset+1));
                ptr.length--;
                i--;
                continue;
            }
            if(c=='\u009c') { // string terminator
                foundStartOfString=false;
                // remove
                System.arraycopy(ptr.chars, i+1,
                  ptr.chars, i, ptr.length-(i-ptr.offset+1));
                ptr.length--;
                i--;
                continue;
            }
            type=Character.getType(c);
            switch(type) {
                // turned this off because it broke truncation
                // could be smarter if necessary
//                case Character.CONTROL:
//                    if(!trimTrailingSpaces) {
//                        c2=replace(c);
//                        if(c2==0) {
//                            replaceChar(ptr, i, replaceMultiple(c));
//                            nacoNormalize2007(ptr, keepFirstComma,
trimTrailingSpaces);
//                            return;
//                        }
//                        ptr.chars[i]=c2;
//                        break;
//                    }
                case Character.FORMAT:
                case Character.PRIVATE_USE:
                case Character.SURROGATE:
                case Character.MODIFIER_LETTER:
                case Character.COMBINING_SPACING_MARK:
                case Character.ENCLOSING_MARK:
                case Character.NON_SPACING_MARK:
                    // remove
                    System.arraycopy(ptr.chars, i+1,
                      ptr.chars, i, ptr.length-(i-ptr.offset+1));
                    ptr.length--;
                    i--;
                    break;
                case Character.CONNECTOR_PUNCTUATION:
                case Character.DASH_PUNCTUATION:
                case Character.FINAL_QUOTE_PUNCTUATION:
                case Character.INITIAL_QUOTE_PUNCTUATION:
                case Character.MODIFIER_SYMBOL:
                case Character.LINE_SEPARATOR:
                case Character.PARAGRAPH_SEPARATOR:
                case Character.SPACE_SEPARATOR:
                    // replace with space
                    ptr.chars[i]=' ';
                    break;
                case Character.END_PUNCTUATION:
                    if(c==']') {
                        // remove
                        System.arraycopy(ptr.chars, i+1,
                          ptr.chars, i, ptr.length-(i-ptr.offset+1));
                        ptr.length--;
                        i--;
                        break;
                    }
                    // replace with space
                    ptr.chars[i]=' ';
                    break;
                case Character.OTHER_PUNCTUATION:
                    if(c=='#' || c=='&' || c=='@')
                        break; // retain
                    if(c==',' && keepFirstComma && !foundFirstComma) {
                        foundFirstComma=true;
                        break; // retain
                    }
                    // replace all others with space
                    ptr.chars[i]=' ';
                    break;
                case Character.START_PUNCTUATION:
                    if(c=='[') {
                        // remove
                        System.arraycopy(ptr.chars, i+1,
                          ptr.chars, i, ptr.length-(i-ptr.offset+1));
                        ptr.length--;
                        i--;
                        break;
                    }
                    // replace with space
                    ptr.chars[i]=' ';
                    break;
                case Character.MATH_SYMBOL:
                    if(c=='+' || c=='\u266f') { // musical sharp sign
                        // retain
                        break;
                    }
                    // replace with space
                    ptr.chars[i]=' ';
                    break;
                case Character.OTHER_SYMBOL:
                    if(c=='\u266d') { // musical flat sign
                        // retain
                        break;
                    }
                    // replace with space
                    ptr.chars[i]=' ';
                    break;
                case Character.CURRENCY_SYMBOL:
                case Character.OTHER_LETTER:
                case Character.LETTER_NUMBER:
//                case Character.OTHER_NUMBER:
                    // retain
                    break;
                case Character.DECIMAL_DIGIT_NUMBER:
                case Character.OTHER_NUMBER:
                    ptr.chars[i]=(char)((c%16)+'0');
                    break;
                default:
                    c2=replace(c);
                    if(c2==0) {
                        replaceChar(ptr, i, replaceMultiple(c));
                        nacoNormalize2007(ptr, keepFirstComma,
trimTrailingSpaces);
                        return;
                    }
                    ptr.chars[i]=c2;
            }
        }

        // remove leading, trailing and redundant blanks
        for(int i=ptr.offset; i<ptr.offset+ptr.length; i++) {
            if(ptr.chars[i]==' ') {
                ptr.offset++;
                ptr.length--;
            }
            else
                break;
        }
        if(trimTrailingSpaces)
            for(int i=ptr.offset+ptr.length-1; i>=ptr.offset; i--) {
                if(ptr.chars[i]==' ') {
                    ptr.length--;
                }
                else
                    break;
            }
        for(int i=ptr.offset+1; i<ptr.offset+ptr.length; i++) {
            if(ptr.chars[i]==' ' && ptr.chars[i-1]==' ') { // remove
                System.arraycopy(ptr.chars, i+1,
                  ptr.chars, i, ptr.length-(i-ptr.offset+1));
                ptr.length--;
                i--;
            }
        }
    }


    private static char replace(char c) {
        switch(c) {
//            case '\u2070': // superscript zero
//            case '\u2080': // subscript zero
//                c=ptr.chars[i]='0';
//            case '\u00b9': // superscript one
//                c=ptr.chars[i]='1';
//                continue;
//            case '\u00b2': // superscript two
//                c=ptr.chars[i]='2';
//                continue;
//            case '\u00b3': // superscript three
//                c=ptr.chars[i]='3';
//                continue;
//            case '\u2074': // superscript four
//            case '\u2084': // subscript four
//                c=ptr.chars[i]='4';
//            case '\u2075': // superscript five
//            case '\u2085': // subscript five
//                c=ptr.chars[i]='5';
//            case '\u2076': // superscript six
//            case '\u2086': // subscript six
//                c=ptr.chars[i]='6';
//            case '\u2077': // superscript seven
//            case '\u2087': // subscript seven
//                c=ptr.chars[i]='7';
//            case '\u2078': // superscript eight
//            case '\u2088': // subscript eight
//                c=ptr.chars[i]='8';
//            case '\u2079': // superscript nine
//            case '\u2089': // subscript nine
//                c=ptr.chars[i]='9';
            case '\u00bc': // one quarter
            case '\u00bd': // one half
            case '\u00be': // three quarters
            case '\u00c6': // uppercase digraph ae
            case '\u00e6': // lowercase digraph ae
            case '\u00de': // uppercase thorn
            case '\u00df': // esszett
            case '\u00fe': // lowercase thorn
            case '\u0132': // uppercase digraph ij
            case '\u0133': // lowercase digraph ij
            case '\u0152': // uppercase digraph oe
            case '\u0153': // lowercase digraph oe
            case '\u0195': // lowercase digraph hv
            case '\u01a2': // uppercase digraph oi
            case '\u01a3': // lowercase digraph oi
            case '\u01be': // lowercase digraph ts
            case '\u01c4': // uppercase letter dz with caron
            case '\u01c5': // titlecase letter dz with caron
            case '\u01c6': // lowercase letter dz with caron
            case '\u01c7': // uppercase letter lj
            case '\u01c8': // titlecase letter lj
            case '\u01c9': // lowercase letter lj
            case '\u01ca': // uppercase letter nj
            case '\u01cb': // titlecase letter nj
            case '\u01cc': // lowercase letter nj
            case '\u01e2': // uppercase digraph ae with macron
            case '\u01e3': // lowercase digraph ae with macron
            case '\u01f1': // uppercase letter dz
            case '\u01f2': // titlecase letter dz
            case '\u01f3': // lowercase letter dz
            case '\u01fc': // uppercase digraph ae with acute
            case '\u01fd': // lowercase digraph ae with acute
            case '\u0222': // uppercase digraph ou
            case '\u0223': // lowercase digraph ou
            case '\ufb00': // lowercase digraph ff
            case '\ufb01': // lowercase digraph fi
            case '\ufb02': // lowercase digraph fl
            case '\ufb03': // lowercase digraph ffi
            case '\ufb04': // lowercase digraph ffl
            case '\ufb05': // lowercase digraph long st
            case '\ufb06': // lowercase digraph st
            case '\u1F80': // GREEK SMALL LETTER ALPHA WITH PSILI AND
YPOGEGRAMMENI
            case '\u1F81': // GREEK SMALL LETTER ALPHA WITH DASIA AND
YPOGEGRAMMENI
            case '\u1F82': // GREEK SMALL LETTER ALPHA WITH PSILI AND
VARIA AND YPOGEGRAMMENI
            case '\u1F83': // GREEK SMALL LETTER ALPHA WITH DASIA AND
VARIA AND YPOGEGRAMMENI
            case '\u1F84': // GREEK SMALL LETTER ALPHA WITH PSILI AND
OXIA AND YPOGEGRAMMENI
            case '\u1F85': // GREEK SMALL LETTER ALPHA WITH DASIA AND
OXIA AND YPOGEGRAMMENI
            case '\u1F86': // GREEK SMALL LETTER ALPHA WITH PSILI AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1F87': // GREEK SMALL LETTER ALPHA WITH DASIA AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1F88': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
PROSGEGRAMMENI
            case '\u1F89': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
PROSGEGRAMMENI
            case '\u1F8A': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
VARIA AND PROSGEGRAMMENI
            case '\u1F8B': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
VARIA AND PROSGEGRAMMENI
            case '\u1F8C': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
OXIA AND PROSGEGRAMMENI
            case '\u1F8D': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
OXIA AND PROSGEGRAMMENI
            case '\u1F8E': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1F8F': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1F90': // GREEK SMALL LETTER ETA WITH PSILI AND
YPOGEGRAMMENI
            case '\u1F91': // GREEK SMALL LETTER ETA WITH DASIA AND
YPOGEGRAMMENI
            case '\u1F92': // GREEK SMALL LETTER ETA WITH PSILI AND
VARIA AND YPOGEGRAMMENI
            case '\u1F93': // GREEK SMALL LETTER ETA WITH DASIA AND
VARIA AND YPOGEGRAMMENI
            case '\u1F94': // GREEK SMALL LETTER ETA WITH PSILI AND OXIA
AND YPOGEGRAMMENI
            case '\u1F95': // GREEK SMALL LETTER ETA WITH DASIA AND OXIA
AND YPOGEGRAMMENI
            case '\u1F96': // GREEK SMALL LETTER ETA WITH PSILI AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1F97': // GREEK SMALL LETTER ETA WITH DASIA AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1F98': // GREEK CAPITAL LETTER ETA WITH PSILI AND
PROSGEGRAMMENI
            case '\u1F99': // GREEK CAPITAL LETTER ETA WITH DASIA AND
PROSGEGRAMMENI
            case '\u1F9A': // GREEK CAPITAL LETTER ETA WITH PSILI AND
VARIA AND PROSGEGRAMMENI
            case '\u1F9B': // GREEK CAPITAL LETTER ETA WITH DASIA AND
VARIA AND PROSGEGRAMMENI
            case '\u1F9C': // GREEK CAPITAL LETTER ETA WITH PSILI AND
OXIA AND PROSGEGRAMMENI
            case '\u1F9D': // GREEK CAPITAL LETTER ETA WITH DASIA AND
OXIA AND PROSGEGRAMMENI
            case '\u1F9E': // GREEK CAPITAL LETTER ETA WITH PSILI AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1F9F': // GREEK CAPITAL LETTER ETA WITH DASIA AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1FA0': // GREEK SMALL LETTER OMEGA WITH PSILI AND
YPOGEGRAMMENI
            case '\u1FA1': // GREEK SMALL LETTER OMEGA WITH DASIA AND
YPOGEGRAMMENI
            case '\u1FA2': // GREEK SMALL LETTER OMEGA WITH PSILI AND
VARIA AND YPOGEGRAMMENI
            case '\u1FA3': // GREEK SMALL LETTER OMEGA WITH DASIA AND
VARIA AND YPOGEGRAMMENI
            case '\u1FA4': // GREEK SMALL LETTER OMEGA WITH PSILI AND
OXIA AND YPOGEGRAMMENI
            case '\u1FA5': // GREEK SMALL LETTER OMEGA WITH DASIA AND
OXIA AND YPOGEGRAMMENI
            case '\u1FA6': // GREEK SMALL LETTER OMEGA WITH PSILI AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1FA7': // GREEK SMALL LETTER OMEGA WITH DASIA AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1FA8': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
PROSGEGRAMMENI
            case '\u1FA9': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
PROSGEGRAMMENI
            case '\u1FAA': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
VARIA AND PROSGEGRAMMENI
            case '\u1FAB': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
VARIA AND PROSGEGRAMMENI
            case '\u1FAC': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
OXIA AND PROSGEGRAMMENI
            case '\u1FAD': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
OXIA AND PROSGEGRAMMENI
            case '\u1FAE': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1FAF': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1FB3': // GREEK SMALL LETTER ALPHA WITH
YPOGEGRAMMENI
            case '\u1FBC': // GREEK CAPITAL LETTER ALPHA WITH
PROSGEGRAMMENI
            case '\u1FC3': // GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
            case '\u1FCC': // GREEK CAPITAL LETTER ETA WITH
PROSGEGRAMMENI
            case '\u1FF3': // GREEK SMALL LETTER OMEGA WITH
YPOGEGRAMMENI
            case '\u1FFC': // GREEK CAPITAL LETTER OMEGA WITH
PROSGEGRAMMENI
            case '\u1FB2': // GREEK SMALL LETTER ALPHA WITH VARIA AND
YPOGEGRAMMENI
            case '\u1FB4': // GREEK SMALL LETTER ALPHA WITH OXIA AND
YPOGEGRAMMENI
            case '\u1FC2': // GREEK SMALL LETTER ETA WITH VARIA AND
YPOGEGRAMMENI
            case '\u1FC4': // GREEK SMALL LETTER ETA WITH OXIA AND
YPOGEGRAMMENI
            case '\u1FF2': // GREEK SMALL LETTER OMEGA WITH VARIA AND
YPOGEGRAMMENI
            case '\u1FF4': // GREEK SMALL LETTER OMEGA WITH OXIA AND
YPOGEGRAMMENI
            case '\u1FB7': // GREEK SMALL LETTER ALPHA WITH PERISPOMENI
AND YPOGEGRAMMENI
            case '\u1FC7': // GREEK SMALL LETTER ETA WITH PERISPOMENI
AND YPOGEGRAMMENI
            case '\u1FF7': // GREEK SMALL LETTER OMEGA WITH PERISPOMENI
AND YPOGEGRAMMENI
                return 0; // flag that we need to return multiple
characters
            case '\u00c0':
            case '\u00c1':
            case '\u00c2':
            case '\u00c3':
            case '\u00c4':
            case '\u00c5':
            case '\u00e0':
            case '\u00e1':
            case '\u00e2':
            case '\u00e3':
            case '\u00e4':
            case '\u00e5':
            case '\u0100':
            case '\u0101':
            case '\u0102':
            case '\u0103':
            case '\u0104':
            case '\u0105':
            case '\u01cd':
            case '\u01ce':
            case '\u01de':
            case '\u01df':
            case '\u01e0':
            case '\u01e1':
            case '\u01fa':
            case '\u01fb':
            case '\u0200':
            case '\u0201':
            case '\u0202':
            case '\u0203':
            case '\u0226':
            case '\u0227':
            case '\u1e00':
            case '\u1e01':
            case '\u1e9a':
            case '\u1ea0':
            case '\u1ea1':
            case '\u1ea2':
            case '\u1ea3':
            case '\u1ea4':
            case '\u1ea5':
            case '\u1ea6':
            case '\u1ea7':
            case '\u1ea8':
            case '\u1ea9':
            case '\u1eaa':
            case '\u1eab':
            case '\u1eac':
            case '\u1ead':
            case '\u1eae':
            case '\u1eaf':
            case '\u1eb0':
            case '\u1eb1':
            case '\u1eb2':
            case '\u1eb3':
            case '\u1eb4':
            case '\u1eb5':
            case '\u1eb6':
            case '\u1eb7':
                return 'a';
            case '\u0180':
            case '\u0181':
            case '\u0182':
            case '\u0183':
            case '\u0184':
            case '\u0185':
            case '\u1e02':
            case '\u1e03':
            case '\u1e04':
            case '\u1e05':
            case '\u1e06':
            case '\u1e07':
                return 'b';
            case '\u00c7':
            case '\u00e7':
            case '\u0106':
            case '\u0107':
            case '\u0108':
            case '\u0109':
            case '\u010a':
            case '\u010b':
            case '\u010c':
            case '\u010d':
            case '\u0186':
            case '\u0187':
            case '\u0188':
            case '\u1e08':
            case '\u1e09':
                return 'c';
            case '\u00d0': // uppercase eth
            case '\u00f0': // lowercase eth
            case '\u010e':
            case '\u010f':
            case '\u0189':
            case '\u018a':
            case '\u018b':
            case '\u018c':
            case '\u018d':
            case '\u1e0a':
            case '\u1e0b':
            case '\u1e0c':
            case '\u1e0d':
            case '\u1e0e':
            case '\u1e0f':
            case '\u1e10':
            case '\u1e11':
            case '\u1e12':
            case '\u1e13':
            case '\u0110': // uppercase Latin D with stroke
            case '\u0111': // lowercase Latin D with stroke
                return 'd';
            case '\u00c8':
            case '\u00c9':
            case '\u00ca':
            case '\u00cb':
            case '\u00e8':
            case '\u00e9':
            case '\u00ea':
            case '\u00eb':
            case '\u0112':
            case '\u0113':
            case '\u0114':
            case '\u0115':
            case '\u0116':
            case '\u0117':
            case '\u0118':
            case '\u0119':
            case '\u011a':
            case '\u011b':
            case '\u018e':
            case '\u018f':
            case '\u0190':
            case '\u01dd':
            case '\u0204':
            case '\u0205':
            case '\u0206':
            case '\u0207':
            case '\u0228':
            case '\u0229':
            case '\u1e14':
            case '\u1e15':
            case '\u1e16':
            case '\u1e17':
            case '\u1e18':
            case '\u1e19':
            case '\u1e1a':
            case '\u1e1b':
            case '\u1e1c':
            case '\u1e1d':
            case '\u1eb8':
            case '\u1eb9':
            case '\u1eba':
            case '\u1ebb':
            case '\u1ebc':
            case '\u1ebd':
            case '\u1ebe':
            case '\u1ebf':
            case '\u1ec0':
            case '\u1ec1':
            case '\u1ec2':
            case '\u1ec3':
            case '\u1ec4':
            case '\u1ec5':
            case '\u1ec6':
            case '\u1ec7':
                return 'e';
            case '\u0191':
            case '\u0192':
            case '\u1e1e':
            case '\u1e1f':
            case '\u1e9b':
                return 'f';
            case '\u011c':
            case '\u011d':
            case '\u011e':
            case '\u011f':
            case '\u0120':
            case '\u0121':
            case '\u0122':
            case '\u0123':
            case '\u0193':
            case '\u0194':
            case '\u01e4':
            case '\u01e5':
            case '\u01e6':
            case '\u01e7':
            case '\u01f4':
            case '\u01f5':
            case '\u1e20':
            case '\u1e21':
                return 'g';
            case '\u0124':
            case '\u0125':
            case '\u0126':
            case '\u0127':
            case '\u021e':
            case '\u021f':
            case '\u1e22':
            case '\u1e23':
            case '\u1e24':
            case '\u1e25':
            case '\u1e26':
            case '\u1e27':
            case '\u1e28':
            case '\u1e29':
            case '\u1e2a':
            case '\u1e2b':
            case '\u1e96':
                return 'h';
            case '\u00cc':
            case '\u00cd':
            case '\u00ce':
            case '\u00cf':
            case '\u00ec':
            case '\u00ed':
            case '\u00ee':
            case '\u00ef':
            case '\u0128':
            case '\u0129':
            case '\u012a':
            case '\u012b':
            case '\u012c':
            case '\u012d':
            case '\u012e':
            case '\u012f':
            case '\u0130':
            case '\u0131':
            case '\u0196':
            case '\u0197':
            case '\u01cf':
            case '\u01d0':
            case '\u0208':
            case '\u0209':
            case '\u020a':
            case '\u020b':
            case '\u1e2c':
            case '\u1e2d':
            case '\u1e2e':
            case '\u1e2f':
            case '\u1ec8':
            case '\u1ec9':
            case '\u1eca':
            case '\u1ecb':
            case '\u2071': // superscript small letter i
                return 'i';
            case '\u0134':
            case '\u0135':
                return 'j';
            case '\u0136':
            case '\u0137':
            case '\u0138':
            case '\u0198':
            case '\u0199':
            case '\u01e8':
            case '\u01e9':
            case '\u1e30':
            case '\u1e31':
            case '\u1e32':
            case '\u1e33':
            case '\u1e34':
            case '\u1e35':
                return 'k';
            case '\u0139':
            case '\u013a':
            case '\u013b':
            case '\u013c':
            case '\u013d':
            case '\u013e':
            case '\u013f':
            case '\u0140':
            case '\u0141':
            case '\u0142':
            case '\u019a':
            case '\u019b':
            case '\u0234':
            case '\u1e36':
            case '\u1e37':
            case '\u1e38':
            case '\u1e39':
            case '\u1e3a':
            case '\u1e3b':
            case '\u1e3c':
            case '\u1e3d':
            case '\u2113':
                return 'l';
            case '\u019c':
            case '\u1e3e':
            case '\u1e3f':
            case '\u1e40':
            case '\u1e41':
            case '\u1e42':
            case '\u1e43':
                return 'm';
            case '\u00d1':
            case '\u00f1':
            case '\u0143':
            case '\u0144':
            case '\u0145':
            case '\u0146':
            case '\u0147':
            case '\u0148':
            case '\u0149':
            case '\u014a':
            case '\u014b':
            case '\u019d':
            case '\u019e':
            case '\u01f8':
            case '\u01f9':
            case '\u0235':
            case '\u1e44':
            case '\u1e45':
            case '\u1e46':
            case '\u1e47':
            case '\u1e48':
            case '\u1e49':
            case '\u1e4a':
            case '\u1e4b':
            case '\u207f': // superscript latin small letter n
                return 'n';
            case '\u00d2':
            case '\u00d3':
            case '\u00d4':
            case '\u00d5':
            case '\u00d6':
            case '\u00d8':
            case '\u00f2':
            case '\u00f3':
            case '\u00f4':
            case '\u00f5':
            case '\u00f6':
            case '\u00f8':
            case '\u014c':
            case '\u014d':
            case '\u014e':
            case '\u014f':
            case '\u0150':
            case '\u0151':
            case '\u019f':
            case '\u01a0':
            case '\u01a1':
            case '\u01d1':
            case '\u01d2':
            case '\u01fe':
            case '\u01ff':
            case '\u020c':
            case '\u020d':
            case '\u020e':
            case '\u020f':
            case '\u022a':
            case '\u022b':
            case '\u022c':
            case '\u022d':
            case '\u022e':
            case '\u022f':
            case '\u0230':
            case '\u0231':
            case '\u1e4c':
            case '\u1e4d':
            case '\u1e4e':
            case '\u1e4f':
            case '\u1e50':
            case '\u1e51':
            case '\u1e52':
            case '\u1e53':
            case '\u1ecc':
            case '\u1ecd':
            case '\u1ece':
            case '\u1ecf':
            case '\u1ed0':
            case '\u1ed1':
            case '\u1ed2':
            case '\u1ed3':
            case '\u1ed4':
            case '\u1ed5':
            case '\u1ed6':
            case '\u1ed7':
            case '\u1ed8':
            case '\u1ed9':
            case '\u1eda':
            case '\u1edb':
            case '\u1edc':
            case '\u1edd':
            case '\u1ede':
            case '\u1edf':
            case '\u1ee0':
            case '\u1ee1':
            case '\u1ee2':
            case '\u1ee3':
                return 'o';
            case '\u01a4':
            case '\u01a5':
            case '\u1e54':
            case '\u1e55':
            case '\u1e56':
            case '\u1e57':
                return 'p';
            case '\u01ea':
            case '\u01eb':
            case '\u01ec':
            case '\u01ed':
                return 'q';
            case '\u0154':
            case '\u0155':
            case '\u0156':
            case '\u0157':
            case '\u0158':
            case '\u0159':
            case '\u01a6':
            case '\u0210':
            case '\u0211':
            case '\u0212':
            case '\u0213':
            case '\u1e58':
            case '\u1e59':
            case '\u1e5a':
            case '\u1e5b':
            case '\u1e5c':
            case '\u1e5d':
            case '\u1e5e':
            case '\u1e5f':
                return 'r';
            case '\u015a':
            case '\u015b':
            case '\u015c':
            case '\u015d':
            case '\u015e':
            case '\u015f':
            case '\u0160':
            case '\u0161':
            case '\u017f':
            case '\u01a7':
            case '\u01a8':
            case '\u01a9':
            case '\u01aa':
            case '\u0218':
            case '\u0219':
            case '\u1e60':
            case '\u1e61':
            case '\u1e62':
            case '\u1e63':
            case '\u1e64':
            case '\u1e65':
            case '\u1e66':
            case '\u1e67':
            case '\u1e68':
            case '\u1e69':
                return 's';
            case '\u0162':
            case '\u0163':
            case '\u0164':
            case '\u0165':
            case '\u0166':
            case '\u0167':
            case '\u01ab':
            case '\u01ac':
            case '\u01ad':
            case '\u01ae':
            case '\u021a':
            case '\u021b':
            case '\u0236':
            case '\u1e6a':
            case '\u1e6b':
            case '\u1e6c':
            case '\u1e6d':
            case '\u1e6e':
            case '\u1e6f':
            case '\u1e70':
            case '\u1e71':
            case '\u1e97':
                return 't';
            case '\u00d9':
            case '\u00da':
            case '\u00db':
            case '\u00dc':
            case '\u00f9':
            case '\u00fa':
            case '\u00fb':
            case '\u00fc':
            case '\u0168':
            case '\u0169':
            case '\u016a':
            case '\u016b':
            case '\u016c':
            case '\u016d':
            case '\u016e':
            case '\u016f':
            case '\u0170':
            case '\u0171':
            case '\u0172':
            case '\u0173':
            case '\u01af':
            case '\u01b0':
            case '\u01b1':
            case '\u01d3':
            case '\u01d4':
            case '\u01d5':
            case '\u01d6':
            case '\u01d7':
            case '\u01d8':
            case '\u01d9':
            case '\u01da':
            case '\u01db':
            case '\u01dc':
            case '\u0214':
            case '\u0215':
            case '\u0216':
            case '\u0217':
            case '\u1e72':
            case '\u1e73':
            case '\u1e74':
            case '\u1e75':
            case '\u1e76':
            case '\u1e77':
            case '\u1e78':
            case '\u1e79':
            case '\u1e7a':
            case '\u1e7b':
            case '\u1ee4':
            case '\u1ee5':
            case '\u1ee6':
            case '\u1ee7':
            case '\u1ee8':
            case '\u1ee9':
            case '\u1eea':
            case '\u1eeb':
            case '\u1eec':
            case '\u1eed':
            case '\u1eee':
            case '\u1eef':
            case '\u1ef0':
            case '\u1ef1':
                return 'u';
            case '\u01b2':
            case '\u1e7c':
            case '\u1e7d':
            case '\u1e7e':
            case '\u1e7f':
                return 'v';
            case '\u0174':
            case '\u0175':
            case '\u01bf':
            case '\u01f6':
            case '\u01f7':
            case '\u1e80':
            case '\u1e81':
            case '\u1e82':
            case '\u1e83':
            case '\u1e84':
            case '\u1e85':
            case '\u1e86':
            case '\u1e87':
            case '\u1e88':
            case '\u1e89':
            case '\u1e98':
                return 'w';
            case '\u1e8a':
            case '\u1e8b':
            case '\u1e8c':
            case '\u1e8d':
                return 'x';
            case '\u00dd':
            case '\u00fd':
            case '\u00ff':
            case '\u0176':
            case '\u0177':
            case '\u0178':
            case '\u01b3':
            case '\u01b4':
            case '\u0232':
            case '\u0233':
            case '\u1e8e':
            case '\u1e8f':
            case '\u1e99':
            case '\u1ef2':
            case '\u1ef3':
            case '\u1ef4':
            case '\u1ef5':
            case '\u1ef6':
            case '\u1ef7':
            case '\u1ef8':
            case '\u1ef9':
                return 'y';
            case '\u0179':
            case '\u017a':
            case '\u017b':
            case '\u017c':
            case '\u017d':
            case '\u017e':
            case '\u01b5':
            case '\u01b6':
            case '\u01b7':
            case '\u01b8':
            case '\u01b9':
            case '\u01ba':
            case '\u01bb':
            case '\u01ee':
            case '\u01ef':
            case '\u021c':
            case '\u021d':
            case '\u0224':
            case '\u0225':
            case '\u1e90':
            case '\u1e91':
            case '\u1e92':
            case '\u1e93':
            case '\u1e94':
            case '\u1e95':
                return 'z';
            case '\u1F00': //GREEK SMALL LETTER ALPHA WITH PSILI
            case '\u1F01': // GREEK SMALL LETTER ALPHA WITH DASIA
            case '\u1F02': // GREEK SMALL LETTER ALPHA WITH PSILI AND
VARIA
            case '\u1F03': // GREEK SMALL LETTER ALPHA WITH DASIA AND
VARIA
            case '\u1F04': // GREEK SMALL LETTER ALPHA WITH PSILI AND
OXIA
            case '\u1F05': // GREEK SMALL LETTER ALPHA WITH DASIA AND
OXIA
            case '\u1F06': // GREEK SMALL LETTER ALPHA WITH PSILI AND
PERISPOMENI
            case '\u1F07': // GREEK SMALL LETTER ALPHA WITH DASIA AND
PERISPOMENI
            case '\u1F08': // GREEK CAPITAL LETTER ALPHA WITH PSILI
            case '\u1F09': // GREEK CAPITAL LETTER ALPHA WITH DASIA
            case '\u1F0A': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
VARIA
            case '\u1F0B': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
VARIA
            case '\u1F0C': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
OXIA
            case '\u1F0D': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
OXIA
            case '\u1F0E': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
PERISPOMENI
            case '\u1F0F': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
PERISPOMENI
            case '\u1F70': // GREEK SMALL LETTER ALPHA WITH VARIA
            case '\u1F71': // GREEK SMALL LETTER ALPHA WITH OXIA
            case '\u1FB0': // GREEK SMALL LETTER ALPHA WITH VRACHY
            case '\u1FB1': // GREEK SMALL LETTER ALPHA WITH MACRON
            case '\u1FB6': // GREEK SMALL LETTER ALPHA WITH PERISPOMENI
            case '\u1FB8': // GREEK CAPITAL LETTER ALPHA WITH VRACHY
            case '\u1FB9': // GREEK CAPITAL LETTER ALPHA WITH MACRON
            case '\u1FBA': // GREEK CAPITAL LETTER ALPHA WITH VARIA
            case '\u1FBB': // GREEK CAPITAL LETTER ALPHA WITH OXIA
                return '\u03b1'; // lowercase alpha
            case '\u1F10': // GREEK SMALL LETTER EPSILON WITH PSILI
            case '\u1F11': // GREEK SMALL LETTER EPSILON WITH DASIA
            case '\u1F12': // GREEK SMALL LETTER EPSILON WITH PSILI AND
VARIA
            case '\u1F13': // GREEK SMALL LETTER EPSILON WITH DASIA AND
VARIA
            case '\u1F14': // GREEK SMALL LETTER EPSILON WITH PSILI AND
OXIA
            case '\u1F15': // GREEK SMALL LETTER EPSILON WITH DASIA AND
OXIA
            case '\u1F18': // GREEK CAPITAL LETTER EPSILON WITH PSILI
            case '\u1F19': // GREEK CAPITAL LETTER EPSILON WITH DASIA
            case '\u1F1A': // GREEK CAPITAL LETTER EPSILON WITH PSILI
AND VARIA
            case '\u1F1B': // GREEK CAPITAL LETTER EPSILON WITH DASIA
AND VARIA
            case '\u1F1C': // GREEK CAPITAL LETTER EPSILON WITH PSILI
AND OXIA
            case '\u1F1D': // GREEK CAPITAL LETTER EPSILON WITH DASIA
AND OXIA
            case '\u1F72': // GREEK SMALL LETTER EPSILON WITH VARIA
            case '\u1F73': // GREEK SMALL LETTER EPSILON WITH OXIA
            case '\u1FC8': // GREEK CAPITAL LETTER EPSILON WITH VARIA
            case '\u1FC9': // GREEK CAPITAL LETTER EPSILON WITH OXIA
                return '\u03b5'; // lowercase epsilon
            case '\u1F20': // GREEK SMALL LETTER ETA WITH PSILI
            case '\u1F21': // GREEK SMALL LETTER ETA WITH DASIA
            case '\u1F22': // GREEK SMALL LETTER ETA WITH PSILI AND
VARIA
            case '\u1F23': // GREEK SMALL LETTER ETA WITH DASIA AND
VARIA
            case '\u1F24': // GREEK SMALL LETTER ETA WITH PSILI AND OXIA
            case '\u1F25': // GREEK SMALL LETTER ETA WITH DASIA AND OXIA
            case '\u1F26': // GREEK SMALL LETTER ETA WITH PSILI AND
PERISPOMENI
            case '\u1F27': // GREEK SMALL LETTER ETA WITH DASIA AND
PERISPOMENI
            case '\u1F28': // GREEK CAPITAL LETTER ETA WITH PSILI
            case '\u1F29': // GREEK CAPITAL LETTER ETA WITH DASIA
            case '\u1F2A': // GREEK CAPITAL LETTER ETA WITH PSILI AND
VARIA
            case '\u1F2B': // GREEK CAPITAL LETTER ETA WITH DASIA AND
VARIA
            case '\u1F2C': // GREEK CAPITAL LETTER ETA WITH PSILI AND
OXIA
            case '\u1F2D': // GREEK CAPITAL LETTER ETA WITH DASIA AND
OXIA
            case '\u1F2E': // GREEK CAPITAL LETTER ETA WITH PSILI AND
PERISPOMENI
            case '\u1F2F': // GREEK CAPITAL LETTER ETA WITH DASIA AND
PERISPOMENI
            case '\u1F74': // GREEK SMALL LETTER ETA WITH VARIA
            case '\u1F75': // GREEK SMALL LETTER ETA WITH OXIA
            case '\u1FC6': // GREEK SMALL LETTER ETA WITH PERISPOMENI
            case '\u1FCA': // GREEK CAPITAL LETTER ETA WITH VARIA
            case '\u1FCB': // GREEK CAPITAL LETTER ETA WITH OXIA
                return '\u03b7'; // lowercase eta
            case '\u1F30': // GREEK SMALL LETTER IOTA WITH PSILI
            case '\u1F31': // GREEK SMALL LETTER IOTA WITH DASIA
            case '\u1F32': // GREEK SMALL LETTER IOTA WITH PSILI AND
VARIA
            case '\u1F33': // GREEK SMALL LETTER IOTA WITH DASIA AND
VARIA
            case '\u1F34': // GREEK SMALL LETTER IOTA WITH PSILI AND
OXIA
            case '\u1F35': // GREEK SMALL LETTER IOTA WITH DASIA AND
OXIA
            case '\u1F36': // GREEK SMALL LETTER IOTA WITH PSILI AND
PERISPOMENI
            case '\u1F37': // GREEK SMALL LETTER IOTA WITH DASIA AND
PERISPOMENI
            case '\u1F38': // GREEK CAPITAL LETTER IOTA WITH PSILI
            case '\u1F39': // GREEK CAPITAL LETTER IOTA WITH DASIA
            case '\u1F3A': // GREEK CAPITAL LETTER IOTA WITH PSILI AND
VARIA
            case '\u1F3B': // GREEK CAPITAL LETTER IOTA WITH DASIA AND
VARIA
            case '\u1F3C': // GREEK CAPITAL LETTER IOTA WITH PSILI AND
OXIA
            case '\u1F3D': // GREEK CAPITAL LETTER IOTA WITH DASIA AND
OXIA
            case '\u1F3E': // GREEK CAPITAL LETTER IOTA WITH PSILI AND
PERISPOMENI
            case '\u1F3F': // GREEK CAPITAL LETTER IOTA WITH DASIA AND
PERISPOMENI
            case '\u1F76': // GREEK SMALL LETTER IOTA WITH VARIA
            case '\u1F77': // GREEK SMALL LETTER IOTA WITH OXIA
            case '\u1FD0': // GREEK SMALL LETTER IOTA WITH VRACHY
            case '\u1FD1': // GREEK SMALL LETTER IOTA WITH MACRON
            case '\u1FD2': // GREEK SMALL LETTER IOTA WITH DIALYTIKA AND
VARIA
            case '\u1FD3': // GREEK SMALL LETTER IOTA WITH DIALYTIKA AND
OXIA
            case '\u1FD6': // GREEK SMALL LETTER IOTA WITH PERISPOMENI
            case '\u1FD7': // GREEK SMALL LETTER IOTA WITH DIALYTIKA AND
PERISPOMENI
            case '\u1FD8': // GREEK CAPITAL LETTER IOTA WITH VRACHY
            case '\u1FD9': // GREEK CAPITAL LETTER IOTA WITH MACRON
            case '\u1FDA': // GREEK CAPITAL LETTER IOTA WITH VARIA
            case '\u1FDB': // GREEK CAPITAL LETTER IOTA WITH OXIA
                return '\u03b9'; // lowercase iota
            case '\u1F40': // GREEK SMALL LETTER OMICRON WITH PSILI
            case '\u1F41': // GREEK SMALL LETTER OMICRON WITH DASIA
            case '\u1F42': // GREEK SMALL LETTER OMICRON WITH PSILI AND
VARIA
            case '\u1F43': // GREEK SMALL LETTER OMICRON WITH DASIA AND
VARIA
            case '\u1F44': // GREEK SMALL LETTER OMICRON WITH PSILI AND
OXIA
            case '\u1F45': // GREEK SMALL LETTER OMICRON WITH DASIA AND
OXIA
            case '\u1F48': // GREEK CAPITAL LETTER OMICRON WITH PSILI
            case '\u1F49': // GREEK CAPITAL LETTER OMICRON WITH DASIA
            case '\u1F4A': // GREEK CAPITAL LETTER OMICRON WITH PSILI
AND VARIA
            case '\u1F4B': // GREEK CAPITAL LETTER OMICRON WITH DASIA
AND VARIA
            case '\u1F4C': // GREEK CAPITAL LETTER OMICRON WITH PSILI
AND OXIA
            case '\u1F4D': // GREEK CAPITAL LETTER OMICRON WITH DASIA
AND OXIA
            case '\u1F78': // GREEK SMALL LETTER OMICRON WITH VARIA
            case '\u1F79': // GREEK SMALL LETTER OMICRON WITH OXIA
            case '\u1FF8': // GREEK CAPITAL LETTER OMICRON WITH VARIA
            case '\u1FF9': // GREEK CAPITAL LETTER OMICRON WITH OXIA
                return '\u03bf'; // lowercase omicron
            case '\u1F50': // GREEK SMALL LETTER UPSILON WITH PSILI
            case '\u1F51': // GREEK SMALL LETTER UPSILON WITH DASIA
            case '\u1F52': // GREEK SMALL LETTER UPSILON WITH PSILI AND
VARIA
            case '\u1F53': // GREEK SMALL LETTER UPSILON WITH DASIA AND
VARIA
            case '\u1F54': // GREEK SMALL LETTER UPSILON WITH PSILI AND
OXIA
            case '\u1F55': // GREEK SMALL LETTER UPSILON WITH DASIA AND
OXIA
            case '\u1F56': // GREEK SMALL LETTER UPSILON WITH PSILI AND
PERISPOMENI
            case '\u1F57': // GREEK SMALL LETTER UPSILON WITH DASIA AND
PERISPOMENI
            case '\u1F59': // GREEK CAPITAL LETTER UPSILON WITH DASIA
            case '\u1F5B': // GREEK CAPITAL LETTER UPSILON WITH DASIA
AND VARIA
            case '\u1F5D': // GREEK CAPITAL LETTER UPSILON WITH DASIA
AND OXIA
            case '\u1F5F': // GREEK CAPITAL LETTER UPSILON WITH DASIA
AND PERISPOMENI
            case '\u1F7A': // GREEK SMALL LETTER UPSILON WITH VARIA
            case '\u1F7B': // GREEK SMALL LETTER UPSILON WITH OXIA
            case '\u1FE0': // GREEK SMALL LETTER UPSILON WITH VRACHY
            case '\u1FE1': // GREEK SMALL LETTER UPSILON WITH MACRON
            case '\u1FE2': // GREEK SMALL LETTER UPSILON WITH DIALYTIKA
AND VARIA
            case '\u1FE3': // GREEK SMALL LETTER UPSILON WITH DIALYTIKA
AND OXIA
            case '\u1FE6': // GREEK SMALL LETTER UPSILON WITH
PERISPOMENI
            case '\u1FE7': // GREEK SMALL LETTER UPSILON WITH DIALYTIKA
AND PERISPOMENI
            case '\u1FE8': // GREEK CAPITAL LETTER UPSILON WITH VRACHY
            case '\u1FE9': // GREEK CAPITAL LETTER UPSILON WITH MACRON
            case '\u1FEA': // GREEK CAPITAL LETTER UPSILON WITH VARIA
            case '\u1FEB': // GREEK CAPITAL LETTER UPSILON WITH OXIA
                return '\u03c5'; // lowercase upsilon
            case '\u1F60': // GREEK SMALL LETTER OMEGA WITH PSILI
            case '\u1F61': // GREEK SMALL LETTER OMEGA WITH DASIA
            case '\u1F62': // GREEK SMALL LETTER OMEGA WITH PSILI AND
VARIA
            case '\u1F63': // GREEK SMALL LETTER OMEGA WITH DASIA AND
VARIA
            case '\u1F64': // GREEK SMALL LETTER OMEGA WITH PSILI AND
OXIA
            case '\u1F65': // GREEK SMALL LETTER OMEGA WITH DASIA AND
OXIA
            case '\u1F66': // GREEK SMALL LETTER OMEGA WITH PSILI AND
PERISPOMENI
            case '\u1F67': // GREEK SMALL LETTER OMEGA WITH DASIA AND
PERISPOMENI
            case '\u1F68': // GREEK CAPITAL LETTER OMEGA WITH PSILI
            case '\u1F69': // GREEK CAPITAL LETTER OMEGA WITH DASIA
            case '\u1F6A': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
VARIA
            case '\u1F6B': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
VARIA
            case '\u1F6C': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
OXIA
            case '\u1F6D': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
OXIA
            case '\u1F6E': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
PERISPOMENI
            case '\u1F6F': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
PERISPOMENI
            case '\u1F7C': // GREEK SMALL LETTER OMEGA WITH VARIA
            case '\u1F7D': // GREEK SMALL LETTER OMEGA WITH OXIA
            case '\u1FF6': // GREEK SMALL LETTER OMEGA WITH PERISPOMENI
            case '\u1FFA': // GREEK CAPITAL LETTER OMEGA WITH VARIA
            case '\u1FFB': // GREEK CAPITAL LETTER OMEGA WITH OXIA
                return '\u03c9'; // lowercase omega
            case '\u1FE4': // GREEK SMALL LETTER RHO WITH PSILI
            case '\u1FE5': // GREEK SMALL LETTER RHO WITH DASIA
            case '\u1FEC': // GREEK CAPITAL LETTER RHO WITH DASIA
                return '\u03c1'; // lowercase rho
        }
        return Character.toLowerCase(c);
    }

    private static void replaceChar(Pointer ptr, int offset, String s) {
        char[] chars=new char[ptr.length+s.length()-1];
        System.arraycopy(ptr.chars, ptr.offset, chars, 0,
offset-ptr.offset);
        System.arraycopy(s.toCharArray(), 0, chars, offset-ptr.offset,
s.length());
        if(offset+1<ptr.length)
            System.arraycopy(ptr.chars, offset+1, chars,
offset-ptr.offset+s.length(), ptr.length-offset-1);
        ptr.reset(chars);
        return;
    }
    private static String replaceMultiple(char c) {
        switch(c) {
            case '\u00bc': // one quarter
                return "1/4";
            case '\u00bd': // one half
                return "1/2";
            case '\u00be': // three quarters
                return "3/4";
            case '\u00c6': // uppercase digraph ae
            case '\u00e6': // lowercase digraph ae
            case '\u01e2': // uppercase digraph ae with macron
            case '\u01e3': // lowercase digraph ae with macron
            case '\u01fc': // uppercase digraph ae with acute
            case '\u01fd': // lowercase digraph ae with acute
                return "ae";
            case '\u01c4': // uppercase letter dz with caron
            case '\u01c5': // titlecase letter dz with caron
            case '\u01c6': // lowercase letter dz with caron
            case '\u01f1': // uppercase letter dz
            case '\u01f2': // titlecase letter dz
            case '\u01f3': // lowercase letter dz
                return "dz";
            case '\u0195': // lowercase digraph hv
                return "hv";
            case '\u0132': // uppercase digraph ij
            case '\u0133': // lowercase digraph ij
                return "ij";
            case '\u01c7': // uppercase letter lj
            case '\u01c8': // titlecase letter lj
            case '\u01c9': // lowercase letter lj
               return "lj";
            case '\u01ca': // uppercase letter nj
            case '\u01cb': // titlecase letter nj
            case '\u01cc': // lowercase letter nj
                return "nj";
            case '\u0152': // uppercase digraph oe
            case '\u0153': // lowercase digraph oe
                return "oe";
            case '\u01a2': // uppercase digraph oi
            case '\u01a3': // lowercase digraph oi
                return "oi";
            case '\u0222': // uppercase digraph ou
            case '\u0223': // lowercase digraph ou
                return "ou";
            case '\u00df': // esszett
                return "ss";
            case '\u00de': // uppercase thorn
            case '\u00fe': // lowercase thorn
                return "th";
            case '\u01be': // lowercase digraph ts
                return "ts";
            case '\ufb00': // lowercase digraph ff
                return "ff";
            case '\ufb01': // lowercase digraph fi
                return "fi";
            case '\ufb02': // lowercase digraph fl
                return "fl";
            case '\ufb03': // lowercase digraph ffi
                return "ffi";
            case '\ufb04': // lowercase digraph ffl
                return "ffl";
            case '\ufb05': // lowercase digraph long st
            case '\ufb06': // lowercase digraph st
                return "st";
            case '\u1F80': // GREEK SMALL LETTER ALPHA WITH PSILI AND
YPOGEGRAMMENI
            case '\u1F81': // GREEK SMALL LETTER ALPHA WITH DASIA AND
YPOGEGRAMMENI
            case '\u1F82': // GREEK SMALL LETTER ALPHA WITH PSILI AND
VARIA AND YPOGEGRAMMENI
            case '\u1F83': // GREEK SMALL LETTER ALPHA WITH DASIA AND
VARIA AND YPOGEGRAMMENI
            case '\u1F84': // GREEK SMALL LETTER ALPHA WITH PSILI AND
OXIA AND YPOGEGRAMMENI
            case '\u1F85': // GREEK SMALL LETTER ALPHA WITH DASIA AND
OXIA AND YPOGEGRAMMENI
            case '\u1F86': // GREEK SMALL LETTER ALPHA WITH PSILI AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1F87': // GREEK SMALL LETTER ALPHA WITH DASIA AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1F88': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
PROSGEGRAMMENI
            case '\u1F89': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
PROSGEGRAMMENI
            case '\u1F8A': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
VARIA AND PROSGEGRAMMENI
            case '\u1F8B': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
VARIA AND PROSGEGRAMMENI
            case '\u1F8C': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
OXIA AND PROSGEGRAMMENI
            case '\u1F8D': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
OXIA AND PROSGEGRAMMENI
            case '\u1F8E': // GREEK CAPITAL LETTER ALPHA WITH PSILI AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1F8F': // GREEK CAPITAL LETTER ALPHA WITH DASIA AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1FB3': // GREEK SMALL LETTER ALPHA WITH
YPOGEGRAMMENI
            case '\u1FBC': // GREEK CAPITAL LETTER ALPHA WITH
PROSGEGRAMMENI
            case '\u1FB2': // GREEK SMALL LETTER ALPHA WITH VARIA AND
YPOGEGRAMMENI
            case '\u1FB4': // GREEK SMALL LETTER ALPHA WITH OXIA AND
YPOGEGRAMMENI
            case '\u1FB7': // GREEK SMALL LETTER ALPHA WITH PERISPOMENI
AND YPOGEGRAMMENI
                return "\u03b1\u03b9"; // lowercase alpha lowercase iota
            case '\u1F90': // GREEK SMALL LETTER ETA WITH PSILI AND
YPOGEGRAMMENI
            case '\u1F91': // GREEK SMALL LETTER ETA WITH DASIA AND
YPOGEGRAMMENI
            case '\u1F92': // GREEK SMALL LETTER ETA WITH PSILI AND
VARIA AND YPOGEGRAMMENI
            case '\u1F93': // GREEK SMALL LETTER ETA WITH DASIA AND
VARIA AND YPOGEGRAMMENI
            case '\u1F94': // GREEK SMALL LETTER ETA WITH PSILI AND OXIA
AND YPOGEGRAMMENI
            case '\u1F95': // GREEK SMALL LETTER ETA WITH DASIA AND OXIA
AND YPOGEGRAMMENI
            case '\u1F96': // GREEK SMALL LETTER ETA WITH PSILI AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1F97': // GREEK SMALL LETTER ETA WITH DASIA AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1F98': // GREEK CAPITAL LETTER ETA WITH PSILI AND
PROSGEGRAMMENI
            case '\u1F99': // GREEK CAPITAL LETTER ETA WITH DASIA AND
PROSGEGRAMMENI
            case '\u1F9A': // GREEK CAPITAL LETTER ETA WITH PSILI AND
VARIA AND PROSGEGRAMMENI
            case '\u1F9B': // GREEK CAPITAL LETTER ETA WITH DASIA AND
VARIA AND PROSGEGRAMMENI
            case '\u1F9C': // GREEK CAPITAL LETTER ETA WITH PSILI AND
OXIA AND PROSGEGRAMMENI
            case '\u1F9D': // GREEK CAPITAL LETTER ETA WITH DASIA AND
OXIA AND PROSGEGRAMMENI
            case '\u1F9E': // GREEK CAPITAL LETTER ETA WITH PSILI AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1F9F': // GREEK CAPITAL LETTER ETA WITH DASIA AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1FC3': // GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
            case '\u1FCC': // GREEK CAPITAL LETTER ETA WITH
PROSGEGRAMMENI
            case '\u1FC2': // GREEK SMALL LETTER ETA WITH VARIA AND
YPOGEGRAMMENI
            case '\u1FC4': // GREEK SMALL LETTER ETA WITH OXIA AND
YPOGEGRAMMENI
            case '\u1FC7': // GREEK SMALL LETTER ETA WITH PERISPOMENI
AND YPOGEGRAMMENI
                return "\u03b7\u03b9"; //lowercase eta lowercase iota
            case '\u1FA0': // GREEK SMALL LETTER OMEGA WITH PSILI AND
YPOGEGRAMMENI
            case '\u1FA1': // GREEK SMALL LETTER OMEGA WITH DASIA AND
YPOGEGRAMMENI
            case '\u1FA2': // GREEK SMALL LETTER OMEGA WITH PSILI AND
VARIA AND YPOGEGRAMMENI
            case '\u1FA3': // GREEK SMALL LETTER OMEGA WITH DASIA AND
VARIA AND YPOGEGRAMMENI
            case '\u1FA4': // GREEK SMALL LETTER OMEGA WITH PSILI AND
OXIA AND YPOGEGRAMMENI
            case '\u1FA5': // GREEK SMALL LETTER OMEGA WITH DASIA AND
OXIA AND YPOGEGRAMMENI
            case '\u1FA6': // GREEK SMALL LETTER OMEGA WITH PSILI AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1FA7': // GREEK SMALL LETTER OMEGA WITH DASIA AND
PERISPOMENI AND YPOGEGRAMMENI
            case '\u1FA8': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
PROSGEGRAMMENI
            case '\u1FA9': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
PROSGEGRAMMENI
            case '\u1FAA': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
VARIA AND PROSGEGRAMMENI
            case '\u1FAB': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
VARIA AND PROSGEGRAMMENI
            case '\u1FAC': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
OXIA AND PROSGEGRAMMENI
            case '\u1FAD': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
OXIA AND PROSGEGRAMMENI
            case '\u1FAE': // GREEK CAPITAL LETTER OMEGA WITH PSILI AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1FAF': // GREEK CAPITAL LETTER OMEGA WITH DASIA AND
PERISPOMENI AND PROSGEGRAMMENI
            case '\u1FF3': // GREEK SMALL LETTER OMEGA WITH
YPOGEGRAMMENI
            case '\u1FFC': // GREEK CAPITAL LETTER OMEGA WITH
PROSGEGRAMMENI
            case '\u1FF2': // GREEK SMALL LETTER OMEGA WITH VARIA AND
YPOGEGRAMMENI
            case '\u1FF4': // GREEK SMALL LETTER OMEGA WITH OXIA AND
YPOGEGRAMMENI
            case '\u1FF7': // GREEK SMALL LETTER OMEGA WITH PERISPOMENI
AND YPOGEGRAMMENI
                return "\u03c9\u03b9"; //lowercase omega lowercase iota
        }
        return null;
    }
}
class Pointer {
    public byte[] data;
    public char[] chars=null;
    public int    hash, length, offset;

    public Pointer() {
    }

    public Pointer(byte[] data) {
        reset(data, 0, data.length);
    }

    public Pointer(byte[] data, int offset) {
        reset(data, offset, 0);
    }

    public Pointer(byte[] data, int offset, int length) {
        reset(data, offset, length);
    }

    public Pointer(String str) {
        reset(str.toCharArray());
    }

    public Pointer(char[] data) {
        reset(data, 0, data.length);
    }

    public Pointer(char[] data, int offset) {
        reset(data, offset, 0);
    }

    public Pointer(char[] data, int offset, int length) {
        reset(data, offset, length);
    }

    @Override
    public Object clone() {
        try{return super.clone();}
        catch(CloneNotSupportedException e) {
            System.out.println(e);
            return null;
        }
    }

    public Pointer copy() {
        return (Pointer)clone();
    }

    public void endOffset(int endOffset) throws IllegalArgumentException
{
        if(endOffset<offset)
            throw new IllegalArgumentException("new end ("+endOffset+")
less than start("+offset+")");
        length=endOffset-offset+1;
    }

    @Override
    public boolean equals(Object o) {
        if(!(o instanceof Pointer))
            return false;
        Pointer p=(Pointer)o;
        if(length!=p.length) {
            return false;
        }
        int i, j;
        if(data!=null) {
            for(i=offset, j=p.offset; i<offset+length; i++, j++)
                if(data[i]!=p.data[j])
                    return false;
        }
        else 
            for(i=offset, j=p.offset; i<offset+length; i++, j++)
                if(chars[i]!=p.chars[j])
                    return false;
        return true;
    }

    public void free() {
        data=null;
    }


    public byte[] getByteData() {
        byte[] bytes=new byte[length];
        System.arraycopy(this.data, offset, bytes, 0, length);
        return bytes;
    }


    @Override
    public int hashCode()
    {
        int i = hash;
        if(i == 0) {
            char ac[]=chars;
            int  j=offset, k=length;
            for(int l = 0; l < k; l++)
                i=31*i+ac[j++];
            hash = i;
        }
        return i;
    }


    public int increment(int amount) {
        length+=amount;
        return length;
    }

    public int indexOf(char c) {
        char[] a=new char[1];
        a[0]=c;
        return indexOf(chars, this.offset, length, a, 0, 1, 0);
    }

    public int indexOf(Pointer s) {
        return indexOf(chars, this.offset, length, s.chars, s.offset,
                       s.length, 0);
    }

    public int indexOf(Pointer s, int offset) {
        return indexOf(chars, this.offset, length, s.chars, s.offset,
                       s.length, offset);
    }

    public static int indexOf(char str[], int strOffset, int strLength,
      char frag[], int fragOffset, int fragLength, int startOffset) {
        if(startOffset >= strLength)
            return fragLength != 0 ? -1 : strLength;
        if(startOffset < 0)
            startOffset = 0;
        if(fragLength == 0)
            return startOffset;
        char c = frag[fragOffset];
        int j1 = strOffset + startOffset;
        int k1 = strOffset + (strLength - fragLength);
label0:
        do
        {
            while(j1 <= k1 && str[j1] != c) 
                j1++;
            if(j1 > k1)
                return -1;
            int l1 = j1 + 1;
            int i2 = (l1 + fragLength) - 1;
            int j2 = fragOffset + 1;
            while(l1 < i2) 
                if(str[l1++] != frag[j2++])
                {
                    j1++;
                    continue label0;
                }
            return j1 - strOffset;
        } while(true);
    }

    public void reset() {
        reset((byte[])null, 0, 0);
    }

    public void reset(byte[] data) {
        reset(data, 0, data.length);
    }

    public void reset(byte[] data, int offset) {
        reset(data, offset, 0);
    }

    public final void reset(byte[] data, int offset, int length) {
        hash=0;
        this.data=data;
        this.chars=null;
        this.offset=offset;
        this.length=length;
    }

    public void reset(String str) {
        reset(str.toCharArray());
    }

    public final void reset(char[] data) {
        reset(data, 0, data.length);
    }

    public void reset(char[] data, int offset) {
        reset(data, offset, 0);
    }

    public final void reset(char[] data, int offset, int length) {
        hash=0;
        this.data=null;
        this.chars=data;
        this.offset=offset;
        this.length=length;
    }

    public void reset(Pointer ptr) {
        if(ptr.data!=null)
            reset(ptr.data, ptr.offset, ptr.length);
        else
            reset(ptr.chars, ptr.offset, ptr.length);
    }


    public void squeezeout(int offset) {
        squeezeout(offset, 1);
    }


    public void squeezeout(int offset, int length) {
        if(offset<this.offset || offset+length>this.offset+this.length)
            throw new IllegalArgumentException("offset: "+offset+" plus
length: "+length+" out of range:
"+this.offset+"-"+(this.offset+this.length-1));
        System.arraycopy(chars, offset+length, chars, offset,
this.length-offset-length);
        this.length-=length;
    }


    @Override
    public String toString() {
        StringBuilder sb=new StringBuilder();
        sb.append("pointer: offset=").append(offset)
          .append(", length=").append(length);
        if(data!=null)
            sb.append(Util.byteArrayToString(data, offset, length));
        if(chars!=null)
            sb.append('\n').append(new String(chars, offset, length));
        return sb.toString();
    }


    public String toStringVar() {
        if(chars==null)
            return null;
        return new String(chars, offset, length);
    }


    public String toBriefString() {
        StringBuilder sb=new StringBuilder();
        sb.append("pointer: ")
            .append("length=").append(length);
        if(data!=null) {
            if (length > 16) { sb.append("\n"); }
            sb.append(Util.byteArrayToString(data, offset, length));
        }
        return sb.toString();
    }
}

-----Original Message-----
From: Code for Libraries [mailto:[log in to unmask]] On Behalf Of
Bill Dueber
Sent: Wednesday, April 11, 2012 11:27 AM
To: [log in to unmask]
Subject: Modern NACO Normalization (esp. in java?)

I'm about to embark on trying to write code to apply NACO normalization
to
strings (not for field-to-field comparisons, but for correctly sorting
things). I was drivin to this by a complaint about how some Arabic
manuscript titles are sorting.

My end goal is a Solr filter, so I'm most interested in Java code.

It doesn't look "hard" so much as "long and error-prone" so I'm hoping
someone has already done this (or at least has a character map that I
can
easily convert to java).

I've seen the code at the
OCLC<http://www.oclc.org/research/activities/naco/default.htm>,
but it's 10 years old and doesn't have a lot of the non-latin stuff in
it.

Evergreen has a perl
implementation<http://git.evergreen-ils.org/?p=Evergreen.git;a=blob;f=Op
en-ILS/src/perlmods/lib/OpenILS/Utils/Normalize.pm>:
that's probably where I'll start if no one has anything else.

Anyone?


-- 
Bill Dueber
Library Systems Programmer
University of Michigan Library

Top of Message | Previous Page | Permalink

Advanced Options


Options

Log In

Log In

Get Password

Get Password


Search Archives

Search Archives


Subscribe or Unsubscribe

Subscribe or Unsubscribe


Archives

March 2024
February 2024
January 2024
December 2023
November 2023
October 2023
September 2023
August 2023
July 2023
June 2023
May 2023
April 2023
March 2023
February 2023
January 2023
December 2022
November 2022
October 2022
September 2022
August 2022
July 2022
June 2022
May 2022
April 2022
March 2022
February 2022
January 2022
December 2021
November 2021
October 2021
September 2021
August 2021
July 2021
June 2021
May 2021
April 2021
March 2021
February 2021
January 2021
December 2020
November 2020
October 2020
September 2020
August 2020
July 2020
June 2020
May 2020
April 2020
March 2020
February 2020
January 2020
December 2019
November 2019
October 2019
September 2019
August 2019
July 2019
June 2019
May 2019
April 2019
March 2019
February 2019
January 2019
December 2018
November 2018
October 2018
September 2018
August 2018
July 2018
June 2018
May 2018
April 2018
March 2018
February 2018
January 2018
December 2017
November 2017
October 2017
September 2017
August 2017
July 2017
June 2017
May 2017
April 2017
March 2017
February 2017
January 2017
December 2016
November 2016
October 2016
September 2016
August 2016
July 2016
June 2016
May 2016
April 2016
March 2016
February 2016
January 2016
December 2015
November 2015
October 2015
September 2015
August 2015
July 2015
June 2015
May 2015
April 2015
March 2015
February 2015
January 2015
December 2014
November 2014
October 2014
September 2014
August 2014
July 2014
June 2014
May 2014
April 2014
March 2014
February 2014
January 2014
December 2013
November 2013
October 2013
September 2013
August 2013
July 2013
June 2013
May 2013
April 2013
March 2013
February 2013
January 2013
December 2012
November 2012
October 2012
September 2012
August 2012
July 2012
June 2012
May 2012
April 2012
March 2012
February 2012
January 2012
December 2011
November 2011
October 2011
September 2011
August 2011
July 2011
June 2011
May 2011
April 2011
March 2011
February 2011
January 2011
December 2010
November 2010
October 2010
September 2010
August 2010
July 2010
June 2010
May 2010
April 2010
March 2010
February 2010
January 2010
December 2009
November 2009
October 2009
September 2009
August 2009
July 2009
June 2009
May 2009
April 2009
March 2009
February 2009
January 2009
December 2008
November 2008
October 2008
September 2008
August 2008
July 2008
June 2008
May 2008
April 2008
March 2008
February 2008
January 2008
December 2007
November 2007
October 2007
September 2007
August 2007
July 2007
June 2007
May 2007
April 2007
March 2007
February 2007
January 2007
December 2006
November 2006
October 2006
September 2006
August 2006
July 2006
June 2006
May 2006
April 2006
March 2006
February 2006
January 2006
December 2005
November 2005
October 2005
September 2005
August 2005
July 2005
June 2005
May 2005
April 2005
March 2005
February 2005
January 2005
December 2004
November 2004
October 2004
September 2004
August 2004
July 2004
June 2004
May 2004
April 2004
March 2004
February 2004
January 2004
December 2003
November 2003

ATOM RSS1 RSS2



LISTS.CLIR.ORG

CataList Email List Search Powered by the LISTSERV Email List Manager