root/apps/outlook/branches/1.5/DotTelSystem/Punycode/generate/GenerateNFKC.cs @ 580

/// <summary> Copyright (C) 2004, 2007  Free Software Foundation, Inc.
/// *
/// Author: Alexander Gnauck AG-Software
/// *
/// This file is part of GNU Libidn.
/// *
/// This library is free software; you can redistribute it and/or
/// modify it under the terms of the GNU Lesser General Public License
/// as published by the Free Software Foundation; either version 2.1 of
/// the License, or (at your option) any later version.
/// *
/// This library is distributed in the hope that it will be useful, but
/// WITHOUT ANY WARRANTY; without even the implied warranty of
/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
/// Lesser General Public License for more details.
/// *
/// You should have received a copy of the GNU Lesser General Public
/// License along with this library; if not, write to the Free Software
/// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
/// USA
/// </summary>
using System;
using System.IO;
using System.Collections;
using System.Text;

namespace gnu.inet.encoding.misc
{
    public class GenerateNFKC
    {
        internal static string stripComment(string sIn)
        {
            int c = sIn.IndexOf('#');
            if (c == -1)
            {
                return sIn;
            }
            else
            {
                return sIn.Substring(0, (c) - (0));
            }
        }

        internal static string[] split(string sIn, char sep)
        {
            StringBuilder sb = new StringBuilder(sIn);
            int c = 0;
            for (int i = 0; i < sb.Length; i++)
            {
                if (sb[i] == sep)
                {
                    c++;
                }
            }

            string[] sOut = new string[c + 1];
            c = 0;
            int l = 0;
            for (int i = 0; i < sb.Length; i++)
            {
                if (sb[i] == sep)
                {
                    if (l >= i)
                    {
                        sOut[c] = "";
                    }
                    else
                    {
                        // TODO, check this
                        sOut[c] = sb.ToString(l, i-l);
                    }
                    l = i + 1;
                    c++;
                }
            }
            if (l < sb.Length)
            {
                sOut[c] = sb.ToString(l, sb.Length - l);
            }
            return sOut;
        }

        internal static bool isCompatibilityMapping(string sIn)
        {
            return sIn.Length > 0 && sIn[0] == '<';
        }

        internal static string stripCompatibilityTag(string sIn)
        {
            return sIn.Substring(sIn.IndexOf('>') + 2);
        }

        internal static string toString(string sIn)
        {
            StringBuilder sOut = new StringBuilder();
            string[] chars = split(sIn, ' ');
            for (int i = 0; i < chars.Length; i++)
            {
                if (chars[i].Equals("005C"))
                {
                    sOut.Append("\\\\");
                }
                else if (chars[i].Equals("0022"))
                {
                    sOut.Append("\\\"");
                }
                else
                {
                    sOut.Append("\\u");
                    sOut.Append(chars[i]);
                }
            }
            return sOut.ToString();
        }
            
        internal static string decompose(string sIn, SortedList mappings)
        {
            StringBuilder sOut = new StringBuilder();
            string[] c = split(sIn, ' ');

            for (int i = 0; i < c.Length; i++)
            {
                if (mappings.ContainsKey(c[i]))
                {
                    if (sOut.Length > 0)
                    {
                        sOut.Append(" ");
                    }
                    sOut.Append(decompose((string)mappings[c[i]], mappings));
                }
                else
                {
                    if (sOut.Length > 0)
                    {
                        sOut.Append(" ");
                    }
                    sOut.Append(c[i]);
                }
            }

            return sOut.ToString();
        }
            
        public static void Generate()
        {
            // Check if the unicode files exist
            {
                FileInfo f1 = new FileInfo("CompositionExclusions.txt");
                FileInfo f2 = new FileInfo("UnicodeData.txt");
                bool tmpBool;
                if (File.Exists(f1.FullName))
                    tmpBool = true;
                else
                    tmpBool = Directory.Exists(f1.FullName);
                bool tmpBool2;
                if (File.Exists(f2.FullName))
                    tmpBool2 = true;
                else
                    tmpBool2 = Directory.Exists(f2.FullName);
                if (!tmpBool || !tmpBool2)
                {
                    Console.WriteLine("Unable to find UnicodeData.txt or CompositionExclusions.txt.");
                    Console.WriteLine("Please download the latest version of these file from:");
                    Console.WriteLine("http://www.unicode.org/Public/UNIDATA/");
                    System.Environment.Exit(1);
                }
            }
                    
            ArrayList exclusions = new ArrayList();
            {            
                StreamReader r = new StreamReader("CompositionExclusions.txt", System.Text.Encoding.Default);
                string line;
                while (null != (line = r.ReadLine()))
                {
                    line = stripComment(line);
                    line = line.Trim();
                    if (line.Length == 0)
                    {
                        // Empty line
                    }
                    else if (line.Length == 4)
                    {
                        exclusions.Add(line);
                    }
                    else
                    {
                        // Skip code points > 0xffff
                    }
                }
                r.Close();
            }

            // Read UnicodeData
            
            SortedList canonical = new SortedList();        
            SortedList compatibility = new SortedList();        
            SortedList combiningClasses = new SortedList();
            {        
                StreamReader r = new StreamReader("UnicodeData.txt", Encoding.Default);
                string line;
                while (null != (line = r.ReadLine()))
                {
                    line = stripComment(line);
                    line = line.Trim();

                    if (line.Length == 0)
                    {
                        // Empty line
                    }
                    else
                    {
                        string[] f = split(line, ';');

                        if (f[0].Length == 4)
                        {
                            if (!f[5].Equals(""))
                            {
                                if (isCompatibilityMapping(f[5]))
                                {
                                    compatibility[f[0]] = stripCompatibilityTag(f[5]);
                                }
                                else
                                {
                                    compatibility[f[0]] = f[5];
                                    if (!exclusions.Contains(f[0]))
                                    {
                                        canonical[f[0]] = f[5];
                                    }
                                }
                            }
                            if (!f[3].Equals("0"))
                            {
                                //UPGRADE_TODO: Method 'java.lang.Integer.parseInt' was converted to 'System.Convert.ToInt32' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073'"
                                combiningClasses[(int)System.Convert.ToInt32(f[0], 16)] = f[3];
                            }
                        }
                        else
                        {
                            // Skip code points > 0xffff
                        }
                    }
                }
                r.Close();
            }

            // Recursively apply compatibility mappings
            while (true)
            {
                bool replaced = false;
                            
                IEnumerator i = new HashSet(compatibility.Keys).GetEnumerator();            
                while (i.MoveNext())
                {                
                    string k = (string)i.Current;
                    string v = (string)compatibility[k];

                    string d = decompose(v, compatibility);
                    if (!d.Equals(v))
                    {
                        replaced = true;
                        compatibility[k] = d;
                    }
                }

                if (!replaced)
                {
                    break;
                }
            }

            // Eliminate duplicate mappings        
            SortedList compatibilityKeys = new SortedList();
            ArrayList compatibilityMappings = new ArrayList();
            {            
                IEnumerator i = new HashSet(compatibility.Keys).GetEnumerator();            
                while (i.MoveNext())
                {                
                    string k = (string)i.Current;
                    string v = (string)compatibility[k];

                    int index = compatibilityMappings.IndexOf(v);
                    if (index == -1)
                    {
                        index = compatibilityMappings.Count;
                        compatibilityMappings.Add(v);
                    }
                    compatibilityKeys[k] = (int)index;
                }
            }

            // Create composition tables        
            SortedList firstMap = new SortedList();        
            SortedList secondMap = new SortedList();
            {            
                IEnumerator i = new HashSet(canonical.Keys).GetEnumerator();            
                while (i.MoveNext())
                {                
                    string k = (string)i.Current;
                    string v = (string)canonical[k];

                    string[] s = split(v, ' ');

                    if (s.Length == 2)
                    {
                        // If both characters have the same combining class, they
                        // won't be combined (in the sequence AB, B is blocked from
                        // A if both have the same combining class)                    
                        string cc1 = (string)combiningClasses[(int)System.Convert.ToInt32(s[0], 16)];                    
                        string cc2 = (string)combiningClasses[(int)System.Convert.ToInt32(s[1], 16)];
                        if (cc1 != null || (cc1 != null && cc1.Equals(cc2)))
                        {
                            // Ignore this composition                        
                            // TODO check this
                            //i.remove();                        
                            canonical.Remove(k);
                            continue;
                        }

                        if (firstMap.ContainsKey(s[0]))
                        {
                            int c = (int)firstMap[s[0]];
                            firstMap[s[0]] = (int)(c + 1);
                        }
                        else
                        {
                            firstMap[s[0]] = 1;
                        }

                        if (secondMap.ContainsKey(s[1]))
                        {
                            int c = (int)secondMap[s[1]];
                            secondMap[s[1]] = (int)(c + 1);
                        }
                        else
                        {
                            secondMap[s[1]] = 1;
                        }
                    }
                    else if (s.Length > 2)
                    {
                        Console.WriteLine("? wrong canonical mapping for " + k);
                        System.Environment.Exit(1);
                    }
                }
            }
                    
            SortedList singleFirstComposition = new SortedList();        
            SortedList singleSecondComposition = new SortedList();        
            SortedList complexComposition = new SortedList();

            int composeLookupMax = 0;
            {            
                IEnumerator i = new HashSet(canonical.Keys).GetEnumerator();         
                while (i.MoveNext())
                {                
                    string k = (string) i.Current;
                    string v = (string) canonical[k];

                    string[] s = split(v, ' ');

                    if (s.Length == 2)
                    {
                        // TODO, check this
                        int first = 0;
                        if(firstMap.Contains(s[0]))
                            first = (int) firstMap[s[0]];
                        
                        int second = 0;
                        if (secondMap.Contains(s[1]))
                            second = (int) secondMap[s[1]];
                        // TODO, check this

                        if (first == 1)
                        {
                            singleFirstComposition[s[0]] = new string[] { s[1], k };                        
                            composeLookupMax = System.Math.Max(composeLookupMax, System.Convert.ToInt32(s[0], 16));
                        }
                        else if (second == 1)
                        {
                            singleSecondComposition[s[1]] = new string[] { s[0], k };                        
                            composeLookupMax = System.Math.Max(composeLookupMax, System.Convert.ToInt32(s[1], 16));
                        }
                        else
                        {
                            if (complexComposition.ContainsKey(s[0]))
                            {                            
                                SortedList m = (SortedList)complexComposition[s[0]];
                                if (m.ContainsKey(s[1]))
                                {
                                    Console.WriteLine("? ambiguous canonical mapping for " + s[0]);
                                    System.Environment.Exit(1);
                                }
                                m[s[1]] = k;
                            }
                            else
                            {                            
                                SortedList m = new SortedList();
                                m[s[1]] = k;
                                complexComposition[s[0]] = m;
                            }                        
                            composeLookupMax = System.Math.Max(composeLookupMax, System.Convert.ToInt32(s[0], 16));                        
                            composeLookupMax = System.Math.Max(composeLookupMax, System.Convert.ToInt32(s[1], 16));
                        }
                    }
                }
            }

            Console.WriteLine("Generating CombiningClass.cs file...");

            // Dump combining classes
            {
                StreamWriter w = new StreamWriter("CombiningClass.cs", false, Encoding.Default);
                w.WriteLine("// Do not edit !!!");
                w.WriteLine("// this file is generated automatically");
                w.WriteLine();            
                w.WriteLine("public class CombiningClass");            
                w.WriteLine("{");            
                w.WriteLine("\tpublic static readonly int[,] c = new int[,] {");
                System.Text.StringBuilder index = new System.Text.StringBuilder();

                int count = 0;

                for (int i = 0; i < 256; i++)
                {
                    bool empty = true;

                    StringBuilder page = new StringBuilder();
                    page.Append("    { /* Page " + i + " */");

                    for (int j = 0; j < 256; j++)
                    {
                        int c = (int)((i << 8) + j);
                        string cc = (string)combiningClasses[c];

                        if (0 == (j & 31))
                        {
                            page.Append("\r\n      ");
                        }
                        if (cc == null)
                        {
                            page.Append("0, ");
                        }
                        else
                        {
                            page.Append(cc + ", ");
                            empty = false;
                        }
                    }
                    page.Append("\r\n    },");

                    index.Append("    ");

                    if (!empty)
                    {                    
                        w.WriteLine(page.ToString());
                        index.Append(count++);
                        index.Append(",\r\n");
                    }
                    else
                    {
                        index.Append("-1,\r\n");
                    }
                }            
                w.WriteLine("  };\r\n");
                            
                w.WriteLine("\tpublic static readonly int[] i = new int[] {");
                w.Write(index.ToString());            
                w.WriteLine("  };");            
                w.WriteLine("}");            
                w.Close();
            }

            //Console.WriteLine(" Ok.");
            Console.WriteLine("Generating DecompositionKeys.cs file...");

            // Dump compatibility decomposition
            {
                StreamWriter w = new StreamWriter("DecompositionKeys.cs", false, Encoding.Default);
                w.WriteLine("// Do not edit !!!");
                w.WriteLine("// this file is generated automatically");         
                w.WriteLine();            
                w.WriteLine("public class DecompositionKeys");            
                w.WriteLine("{");

                w.WriteLine("\tpublic static readonly int[] k = new int[] {");            
                IEnumerator i = new HashSet(compatibilityKeys.Keys).GetEnumerator();            
                while (i.MoveNext())
                {            
                    string k = (string)i.Current;
                    int index = ((int)compatibilityKeys[k]);             
                    w.WriteLine("    '\\u" + k + "', " + index + ",");
                }            
                w.WriteLine("  };");            
                w.WriteLine("}");            
                w.Close();
            }

            //Console.WriteLine(" Ok.");
            Console.WriteLine("Generating DecompositionMappings.cs file...");

            {
                StreamWriter w = new StreamWriter("DecompositionMappings.cs", false, Encoding.Default);
                w.WriteLine("// Do not edit !!!");
                w.WriteLine("// this file is generated automatically");
                w.WriteLine();            
                w.WriteLine("public class DecompositionMappings");            
                w.WriteLine("{");            
                w.WriteLine("\tpublic static readonly string[] m = new string[] {");
                IEnumerator i = compatibilityMappings.GetEnumerator();
                while (i.MoveNext())
                {                
                    string m = (string)i.Current;             
                    w.WriteLine("    \"" + toString(m) + "\",");
                }            
                w.WriteLine("  };");            
                w.WriteLine("}");            
                w.Close();
            }

            //Console.WriteLine(" Ok.");
            Console.WriteLine("Generating Composition.cs file...");

            // Dump canonical composition
            {
                StreamWriter w = new StreamWriter("Composition.cs", false, Encoding.Default);
                w.WriteLine("// Do not edit !!!");
                w.WriteLine("// this file is generated automatically");
                w.WriteLine();            
                w.WriteLine("public class Composition");            
                w.WriteLine("{");

                IEnumerator i;
                int index = 0;
                           
                SortedList indices = new SortedList();
                            
                i = new HashSet(complexComposition.Keys).GetEnumerator();            
                while (i.MoveNext())
                {                
                    string s0 = (string)i.Current;             
                    indices[(int)System.Convert.ToInt32(s0, 16)] = (int)index;
                    index++;
                }

                int multiSecondStart = index;
                w.WriteLine("\t/* jagged Array */");
                w.WriteLine("\tpublic static readonly char[][] multiFirst = new char[][] {");
                //w.WriteLine("  public final static char[][] multiFirst = new char[][] {");
                i = new HashSet(complexComposition.Keys).GetEnumerator();            
                while (i.MoveNext())
                {                
                    string s0 = (string)i.Current;             
                    SortedList m = (SortedList)complexComposition[s0];
                                   
                    SortedList line = new SortedList();
                    int maxIndex = 1;
                                    
                    System.Collections.IEnumerator i2 = new HashSet(m.Keys).GetEnumerator();                
                    while (i2.MoveNext())
                    {                    
                        string s1 = (string)i2.Current;
                        string k = (string)m[s1];
                                         
                        int s1i = (int)System.Convert.ToInt32(s1, 16);

                        if (!indices.ContainsKey(s1i))
                        {
                            indices[s1i] = (int)index;
                            index++;
                        }
                        line[indices[s1i]] = k;
                        maxIndex = System.Math.Max(maxIndex, ((int)indices[s1i]));
                    }

                    w.Write("\tnew char[] { ");
                    for (int j = multiSecondStart; j <= maxIndex; j++)
                    {
                        if (line.ContainsKey((int)j))
                        {
                            string s = (string)line[(int)j];
                            w.Write("'" + toString(s) + "', ");
                        }
                        else
                        {
                            //w.Write("       0, ");
                            w.Write("'" + toString("0000") + "', ");
                        }
                    }                
                    w.WriteLine("},");
                }            
                w.WriteLine("  };");

                int singleFirstStart = index;

                w.WriteLine("\tpublic static readonly char[,] singleFirst = new char[,] {");
                i = new HashSet(singleFirstComposition.Keys).GetEnumerator();            
                while (i.MoveNext())
                {            
                    string k = (string)i.Current;
                    string[] v = ((string[])singleFirstComposition[k]);             
                    w.WriteLine("    { '" + toString(v[0]) + "', '" + toString(v[1]) + "' },");
                                    
                    if (indices.ContainsKey((int)System.Convert.ToInt32(k, 16)))
                    {
                        Console.WriteLine(k + " already indexed!");
                    }
                                    
                    indices[(int)System.Convert.ToInt32(k, 16)] = (int)index;
                    index++;
                }            
                w.WriteLine("  };");

                int singleSecondStart = index;
                            
                w.WriteLine("\tpublic static readonly char[,] singleSecond = new char[,] {");
                i = new HashSet(singleSecondComposition.Keys).GetEnumerator();            
                while (i.MoveNext())
                {                
                    string k = (string)i.Current;
                    string[] v = ((string[])singleSecondComposition[k]);             
                    w.WriteLine("    { '" + toString(v[0]) + "', '" + toString(v[1]) + "' },");
                                    
                    indices[(int)System.Convert.ToInt32(k, 16)] = (int)index;
                    index++;
                }            
                w.WriteLine("  };");
                            
                w.WriteLine("\tpublic static readonly int multiSecondStart = " + multiSecondStart + ";");            
                w.WriteLine("\tpublic static readonly int singleFirstStart = " + singleFirstStart + ";");            
                w.WriteLine("\tpublic static readonly int singleSecondStart = " + singleSecondStart + ";");

                System.Text.StringBuilder compositionPages = new System.Text.StringBuilder();
                            
                w.WriteLine("\tpublic static readonly int[] composePage = new int[] {");
                int pageCount = 0;
                for (int j = 0; j * 256 < composeLookupMax + 255; j++)
                {
                    bool empty = true;
                    StringBuilder page = new StringBuilder();
                    for (int k = 0; k < 256; k++)
                    {
                        if (k % 16 == 0)
                        {
                            page.Append("\r\n      ");
                        }
                        if (indices.ContainsKey((int)(j * 256 + k)))
                        {
                            page.Append(indices[(int)(j * 256 + k)]);
                            page.Append(", ");
                            empty = false;
                        }
                        else
                        {
                            page.Append("-1, ");
                        }
                    }

                    if (empty)
                    {                    
                        w.WriteLine("    -1,");
                    }
                    else
                    {                 
                        w.WriteLine("    " + pageCount + ",");
                        compositionPages.Append("\t{");
                        compositionPages.Append(page);
                        compositionPages.Append("\r\n    },\r\n");
                        pageCount++;
                    }
                }            
                w.WriteLine("  };");
                //w.WriteLine("\t/* jagged Array */");                        
                w.WriteLine("\tpublic static readonly int[,] composeData = new int[,] {");
                w.Write(compositionPages);            
                w.WriteLine("  };");                        
                w.WriteLine("}");            
                w.Close();
            }

            //Console.WriteLine(" Ok.");             
            Console.WriteLine("Finished!");
        }
    }
}