/*
 *******************************************************************************
 * Copyright (C) 2002-2006, International Business Machines Corporation and    *
 * others. All Rights Reserved.                                                *
 *******************************************************************************
 *
 * $Source: /icu/icuhtml/icu.sf.net/docs/eclipse_howto/eclipse3x.html,v 
 com.ibm.icu.dev.test.charset/TestConversion.java,v $ 
 * $Date: 2006/09/18 21:30:45 $ 
 * $Revision: 1.5 $ 
 *
 *******************************************************************************
 */

package com.ibm.icu.dev.test.charset;

import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;
import java.util.Iterator;

import com.ibm.icu.charset.CharsetDecoderICU;
import com.ibm.icu.charset.CharsetProviderICU;
import com.ibm.icu.dev.test.ModuleTest;
import com.ibm.icu.dev.test.TestDataModule.DataMap;
import com.ibm.icu.impl.ICUResourceBundle;


/**
 * This maps to convtest.c which tests the test file for data-driven conversion tests. 
 * 
 */
public class TestConversion extends ModuleTest {
    /**
     * This maps to the C struct of conversion case in convtest.h that stores the
     * data for a conversion test
     * 
     */
    private class ConversionCase {
        int caseNr;                                         // testcase index   
        String option = null;                               // callback options
        CodingErrorAction cbErrorAction = null;             // callback action type

        // data retrieved from a test case conversion.txt
        String charset;                                     // charset
        String unicode;                                     // unicode string
        ByteBuffer bytes;                                   // bytes
        int[] offsets;                                      // offsets
        boolean finalFlush;                                 // flush
        boolean fallbacks;                                  // fallback
        String outErrorCode;                                // errorCode
        String cbopt;                                       // callback 

        // TestGetUnicodeSet variables
        String map;
        String mapnot;
        int which;
    }

    // public methods --------------------------------------------------------

    public static void main(String[] args) throws Exception {
        new TestConversion().run(args);
    }

    public TestConversion() {
        super("com/ibm/icu/dev/data/testdata/", "conversion");
    }

    /*
     * This method maps to the convtest.cpp runIndexedTest() method to run each
     * type of conversion.
     */
    public void processModules() {
        try { 
            int testFromUnicode = 0;
            String testName = t.getName().toString();
            int testToUnicode = 0;
            // Iterate through and get each of the test case to process
            for (Iterator iter = t.getDataIterator(); iter.hasNext();) {
                DataMap testcase = (DataMap) iter.next();

                if (testName.equalsIgnoreCase("toUnicode")) { 
                    TestToUnicode(testcase, testToUnicode);
                    testToUnicode++;
                } else if (testName.equalsIgnoreCase("fromUnicode")) {                    
                    TestFromUnicode(testcase, testFromUnicode);
                    testFromUnicode++;
                } else if (testName.equalsIgnoreCase("getUnicodeSet")) {
                    TestGetUnicodeSet(testcase);
                } else {
                    warnln("Could not load the test cases for conversion");
                    continue;
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    // private methods -------------------------------------------------------

    private void TestToUnicode(DataMap testcase, int caseNr) {
        // create Conversion case to store the test case data
        ConversionCase cc = new ConversionCase();

        try {
            // retrieve test case data
            cc.caseNr = caseNr;
            cc.charset = ((ICUResourceBundle) testcase.getObject("charset"))
                    .getString();
            cc.bytes = ((ICUResourceBundle) testcase.getObject("bytes"))
                    .getBinary();
            cc.unicode = ((ICUResourceBundle) testcase.getObject("unicode"))
                    .getString();
            cc.offsets = ((ICUResourceBundle) testcase.getObject("offsets"))
                    .getIntVector();
            cc.finalFlush = ((ICUResourceBundle) testcase.getObject("flush"))
                    .getUInt() != 0;
            cc.fallbacks = ((ICUResourceBundle) testcase.getObject("fallbacks"))
                    .getUInt() != 0;
            cc.outErrorCode = ((ICUResourceBundle) testcase
                    .getObject("errorCode")).getString();
            cc.cbopt = ((ICUResourceBundle) testcase.getObject("callback"))
                    .getString();
        } catch (Exception e) { 
            errln("Skipping test: error parsing conversion/toUnicode test case " + cc.caseNr);
            return;
        }
//      ----for debugging only 
        logln("\nTestToUnicode[" + caseNr + "] "
                + cc.charset + " ");
        logln("Bytes:");
        printbytes(cc.bytes, cc.bytes.limit());
        logln("");
        logln("Unicode: " + hex(cc.unicode));    
        logln("Callback: (" + cc.cbopt + ")");  
        logln("\n...............................................");

//         ----for debugging only
        
        //This test case is skipped due to limitation in java's API for decoder replacement 
        // { "ibm-1363", :bin{ a2aea2 }, "\u00a1\u001a", :intvector{ 0, 2 }, :int{1}, :int{0}, "", "?", :bin{""} }
        if(cc.caseNr == 63)
        {
            logln("TestToUnicode[" + cc.caseNr + "] " + cc.charset);
            logln("Skipping test due to limitation in Java API - callback replacement value");
            return;
        }
        // process the retrieved test data case
        if (cc.offsets.length == 0) {
            cc.offsets = null;
        } else if (cc.offsets.length != cc.unicode.length()) { 
            errln("Skipping test: toUnicode[" + cc.caseNr + "] unicode[" + cc.unicode.length()
                    + "] and offsets[" + cc.offsets.length
                    + "] must have the same length");
            return;
        }
        // check for the callback replacement value for unmappable
        // characters or malformed errors
        if (cc.cbopt.length() > 0) {
            switch ((cc.cbopt).charAt(0)) {
            case '?':           //CALLBACK_SUBSTITUTE 
                cc.cbErrorAction = CodingErrorAction.REPLACE;
                break;
            case '0':           //CALLBACK_SKIP 
                cc.cbErrorAction = CodingErrorAction.IGNORE;
                break;
            case '.':           //CALLBACK_STOP 
                cc.cbErrorAction = CodingErrorAction.REPORT;
                break;
            case '&':           //CALLBACK_ESCAPE 
                cc.cbErrorAction = CodingErrorAction.REPORT;
                break;
            default:     
                cc.cbErrorAction = null;
                break;
            }
        }
        // check for any options for the callback value 
        cc.option = cc.cbErrorAction == null ? cc.cbopt : cc.cbopt.substring(1);
        if (cc.option == null) {
            cc.option = null;
        }

        logln("TestToUnicode[" + cc.caseNr + "] " + cc.charset);
        ToUnicodeCase(cc);

    }

    private void ToUnicodeCase(ConversionCase cc) {

        // create converter for charset and decoder for each test case
        CharsetProviderICU provider = new CharsetProviderICU();
        CharsetDecoderICU decoder = null;
        Charset charset = null;

        try {
            charset = (Charset) provider.charsetForName(cc.charset);
            decoder = (CharsetDecoderICU) charset.newDecoder();
            decoder.onMalformedInput(CodingErrorAction.REPLACE);
            decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);

        } catch (Exception e) {

            logln("Skipping test:(" + cc.charset
                    + ") due to ICU Charset not supported at this time");
            return;
        }

        // set the callback for the decoder
        if (cc.cbErrorAction != null) {
            decoder.onMalformedInput(cc.cbErrorAction);
            decoder.onUnmappableCharacter(cc.cbErrorAction);

            // set the options (if any: SKIP_STOP_ON_ILLEGAL) for callback
            if (cc.option.equals("i")) {
                decoder.onMalformedInput(CodingErrorAction.REPORT);
            }

            // if callback action is replace, and there is a subchar
            // replace the decoder's default replacement value
            // if substring, skip test due to current api not supporting
            // substring replacement
            if (cc.cbErrorAction.equals(CodingErrorAction.REPLACE)) {
                if (cc.cbopt.length() > 1) {
                    if (cc.cbopt.charAt(1) == '=') {
                        logln("Skipping test due to limitation in Java API - substitution string not supported");

                    } else {
                        // // read NUL-separated subchar first, if any
                        // copy the subchar from Latin-1 characters
                        // start after the NUL
                        if (cc.cbopt.charAt(1) == 0x00) {
                            cc.cbopt = cc.cbopt.substring(2);

                            try {
                                decoder.replaceWith(cc.cbopt);
                            } catch (Exception e) {
                                logln("Skipping test due to limitation in Java API - substitution character sequence size error");

                            }
                        }
                    }
                }
            }
        } 

        // decode source to unicode
        ByteBuffer source = ByteBuffer.wrap(cc.bytes.array());
        CharBuffer out = CharBuffer.allocate((int) (decoder
                .averageCharsPerByte() * source.remaining()));
        do {
            CoderResult cr = decoder.decode(source, out, true);
            if (cr.isOverflow()) {
                int pos = out.position();
                char[] temp = out.array();
                out = CharBuffer.allocate(temp.length * 4);
                out.put(temp);
                out.position(pos);
            } else if (cr.isError()) {
                checkResultsToUnicode(cc,cc.unicode, out); 
                return;
            }
        } while (source.remaining() > 0);

        checkResultsToUnicode(cc,cc.unicode, out);
        return;
    }

    private void TestFromUnicode(DataMap testcase, int caseNr) {

        ConversionCase cc = new ConversionCase();
        cc.caseNr = caseNr;

        try {
            // retrieve test case data
            cc.charset = ((ICUResourceBundle) testcase.getObject("charset"))
                    .getString();
            cc.unicode = ((ICUResourceBundle) testcase.getObject("unicode"))
                    .getString();
            cc.bytes = ((ICUResourceBundle) testcase.getObject("bytes"))
                    .getBinary();
            cc.offsets = ((ICUResourceBundle) testcase.getObject("offsets"))
                    .getIntVector();
            cc.finalFlush = ((ICUResourceBundle) testcase.getObject("flush"))
                    .getUInt() != 0;
            cc.fallbacks = ((ICUResourceBundle) testcase.getObject("fallbacks"))
                    .getUInt() != 0;
            cc.outErrorCode = ((ICUResourceBundle) testcase
                    .getObject("errorCode")).getString();
            cc.cbopt = ((ICUResourceBundle) testcase.getObject("callback"))
                    .getString();


        } catch (Exception e) {
            errln("Skipping test:");
            errln("error parsing conversion/toUnicode test case " + cc.caseNr);
            return;
        }
        // ----for debugging only
        logln("\nTestFromUnicode[" + caseNr + "] "
                + cc.charset + " "); 
        logln("Unicode: " + cc.unicode);
        logln("Bytes:");
        printbytes(cc.bytes, cc.bytes.limit());
        logln("");
        logln("Callback: (" + cc.cbopt + ")"); 
        logln("...............................................");

//         ----for debugging only
    
 
        // TODO: ***Currently skipping test for charset ibm-1390, gb18030,
        // ibm-930 due to external mapping need to be fix
        if (cc.charset.equalsIgnoreCase("ibm-1390")
                || cc.charset.equalsIgnoreCase("gb18030")
                || cc.charset.equalsIgnoreCase("ibm-970")) {
            logln("Skipping test:("
                    + cc.charset
                    + ") due to ICU Charset external mapping not supported at this time");
            return;
        }

        // process the retrieved test data case
        if (cc.offsets.length == 0) {
            cc.offsets = null;
        } else if (cc.offsets.length != cc.bytes.limit()) {
            errln("fromUnicode[" + cc.caseNr + "] bytes[" + cc.bytes
                    + "] and offsets[" + cc.offsets.length
                    + "] must have the same length");
            return;
        }

        // check the callback replacement value
        if (cc.cbopt.length() > 0) {

            switch ((cc.cbopt).charAt(0)) {
            case '?': 
                cc.cbErrorAction = CodingErrorAction.REPLACE;
                break;
            case '0': 
                cc.cbErrorAction = CodingErrorAction.IGNORE;
                break;
            case '.': 
                cc.cbErrorAction = CodingErrorAction.REPORT;
                break;
            case '&': 
                cc.cbErrorAction = CodingErrorAction.REPORT;
                break;
            default: 
                cc.cbErrorAction = null;
                break;
            }

            // check for any options for the callback value -- 
            cc.option = cc.cbErrorAction == null ? cc.cbopt : cc.cbopt.substring(1);
            if (cc.option == null) {
                cc.option = null;
            }
        }
        logln("TestFromUnicode[" + cc.caseNr + "] " + cc.charset);
        FromUnicodeCase(cc);

        return;

    }

    private void FromUnicodeCase(ConversionCase cc) {

        // create charset encoder for conversion test
        CharsetProviderICU provider = new CharsetProviderICU();
        CharsetEncoder encoder = null;
        Charset charset = null;
        try {
            charset = (Charset) provider.charsetForName(cc.charset);
            encoder = (CharsetEncoder) charset.newEncoder();
            encoder.onMalformedInput(CodingErrorAction.REPLACE);
            encoder.onUnmappableCharacter(CodingErrorAction.REPLACE);

        } catch (Exception e) {

            logln("Skipping test:(" + cc.charset
                    + ") due to ICU Charset not supported at this time");
            return;

        }
 
        // set the callback for the encoder 
        if (cc.cbErrorAction != null) {
            encoder.onUnmappableCharacter(cc.cbErrorAction);
            encoder.onMalformedInput(cc.cbErrorAction);

            // if action has an option, put in the option for the case
            if (cc.option.equals("i")) {
                encoder.onMalformedInput(CodingErrorAction.REPORT);
            }

            // if callback action is replace, and there is a subchar
            // replace the decoder's default replacement value
            // if substring, skip test due to current api not supporting
            // substring
            if (cc.cbErrorAction.equals(CodingErrorAction.REPLACE)) {
                if (cc.cbopt.length() > 1) {
                    if (cc.cbopt.length() > 1 && cc.cbopt.charAt(1) == '=') {
                        logln("Skipping test due to limitation in Java API - substitution string not supported");
                        return;
                    } else {
                        // // read NUL-separated subchar first, if any
                        // copy the subchar from Latin-1 characters
                        // start after the NUL
                        if (cc.cbopt.charAt(1) == 0x00) {
                            cc.cbopt = cc.cbopt.substring(2);
                            try {
                                encoder.replaceWith(toByteArray(cc.cbopt));
                            } catch (Exception e) {
                                logln("Skipping test due to limitation in Java API - substitution character sequence size error");
                                return;
                            }
                        }
                    }
                }
            }
        } 
        // do charset encoding from unicode
        CharBuffer uniStr = CharBuffer.wrap(cc.unicode.toCharArray());
        ByteBuffer out = ByteBuffer.allocate((int) (encoder
                .averageBytesPerChar() * uniStr.remaining()));
        do {
            CoderResult cr = encoder.encode(uniStr, out, true);
            if (cr.isOverflow()) {
                int pos = out.position();
                byte[] temp = out.array();
                out = ByteBuffer.allocate(temp.length * 4);
                out.put(temp);
                out.position(pos);
            } else if (cr.isError()) {
                // check the stopped test for current output and match the
                // expected results
                checkResultsFromUnicode(cc,cc.bytes, out);
                return;
            }
            else {
                cr = encoder.flush(out);
            }

        } while (uniStr.remaining() > 0);

        checkResultsFromUnicode(cc,cc.bytes, out);
        return;

    }
    private byte[] toByteArray(String str){
        byte[] ret = new byte[ str.length() ];
        for(int i=0; i<ret.length;i++){
            char ch =  str.charAt(i);
            if(ch<=0xFF){
                ret[i]= (byte)ch;
            }else{
                throw new IllegalArgumentException(" byte value out of range: " + ch);
            }
        }
        return ret;
    }
    private void TestGetUnicodeSet(DataMap testcase) {
        /*
         * charset - will be opened, and ucnv_getUnicodeSet() called on it //
         * map - set of code points and strings that must be in the returned set //
         * mapnot - set of code points and strings that must *not* be in the //
         * returned set // which - numeric UConverterUnicodeSet value Headers {
         * "charset", "map", "mapnot", "which" }
         */
        ConversionCase cc = new ConversionCase();
        // retrieve test case data
        cc.charset = ((ICUResourceBundle) testcase.getObject("charset"))
                .getString();
        cc.map = ((ICUResourceBundle) testcase.getObject("map")).getString();
        cc.mapnot = ((ICUResourceBundle) testcase.getObject("mapnot"))
                .getString();
        cc.which = ((ICUResourceBundle) testcase.getObject("which")).getUInt();
 
        // create charset and encoder for each test case
        logln("Test not supported at this time");

    }

    /**
     * This follows ucnv.c method ucnv_detectUnicodeSignature() to detect the
     * start of the stream for example U+FEFF (the Unicode BOM/signature
     * character) that can be ignored.
     * 
     * Detects Unicode signature byte sequences at the start of the byte stream
     * and returns number of bytes of the BOM of the indicated Unicode charset.
     * 0 is returned when no Unicode signature is recognized.
     * 
     */

    private String detectUnicodeSignature(ByteBuffer source) {
        int signatureLength = 0; // number of bytes of the signature
        final int SIG_MAX_LEN = 5;
        String sigUniCharset = null; // states what unicode charset is the BOM
        int i = 0;

        /*
         * initial 0xa5 bytes: make sure that if we read <SIG_MAX_LEN bytes we
         * don't misdetect something
         */
        byte start[] = { (byte) 0xa5, (byte) 0xa5, (byte) 0xa5, (byte) 0xa5,
                (byte) 0xa5 };

        while (i < source.remaining() && i < SIG_MAX_LEN) {
            start[i] = source.get(i);
            i++;
        }

        if (start[0] == (byte) 0xFE && start[1] == (byte) 0xFF) {
            signatureLength = 2;
            sigUniCharset = "UTF-16BE";
            source.position(signatureLength);
            return sigUniCharset;
        } else if (start[0] == (byte) 0xFF && start[1] == (byte) 0xFE) {
            if (start[2] == (byte) 0x00 && start[3] == (byte) 0x00) {
                signatureLength = 4;
                sigUniCharset = "UTF-32LE";
                source.position(signatureLength);
                return sigUniCharset;
            } else {
                signatureLength = 2;
                sigUniCharset = "UTF-16LE";
                source.position(signatureLength);
                return sigUniCharset;
            }
        } else if (start[0] == (byte) 0xEF && start[1] == (byte) 0xBB
                && start[2] == (byte) 0xBF) {
            signatureLength = 3;
            sigUniCharset = "UTF-8";
            source.position(signatureLength);
            return sigUniCharset;
        } else if (start[0] == (byte) 0x00 && start[1] == (byte) 0x00
                && start[2] == (byte) 0xFE && start[3] == (byte) 0xFF) {
            signatureLength = 4;
            sigUniCharset = "UTF-32BE";
            source.position(signatureLength);
            return sigUniCharset;
        } else if (start[0] == (byte) 0x0E && start[1] == (byte) 0xFE
                && start[2] == (byte) 0xFF) {
            signatureLength = 3;
            sigUniCharset = "SCSU";
            source.position(signatureLength);
            return sigUniCharset;
        } else if (start[0] == (byte) 0xFB && start[1] == (byte) 0xEE
                && start[2] == (byte) 0x28) {
            signatureLength = 3;
            sigUniCharset = "BOCU-1";
            source.position(signatureLength);
            return sigUniCharset;
        } else if (start[0] == (byte) 0x2B && start[1] == (byte) 0x2F
                && start[2] == (byte) 0x76) {

            if (start[3] == (byte) 0x38 && start[4] == (byte) 0x2D) {
                signatureLength = 5;
                sigUniCharset = "UTF-7";
                source.position(signatureLength);
                return sigUniCharset;
            } else if (start[3] == (byte) 0x38 || start[3] == (byte) 0x39
                    || start[3] == (byte) 0x2B || start[3] == (byte) 0x2F) {
                signatureLength = 4;
                sigUniCharset = "UTF-7";
                source.position(signatureLength);
                return sigUniCharset;
            }
        } else if (start[0] == (byte) 0xDD && start[2] == (byte) 0x73
                && start[2] == (byte) 0x66 && start[3] == (byte) 0x73) {
            signatureLength = 4;
            sigUniCharset = "UTF-EBCDIC";
            source.position(signatureLength);
            return sigUniCharset;
        }

        /* no known Unicode signature byte sequence recognized */
        return null;
    }

    void printbytes(ByteBuffer buf, int pos) {
        int cur = buf.position();
        log(" (" + pos + ")==[");
        for (int i = 0; i < pos; i++) {
            log("(" + i + ")" + hex(buf.get(i) & 0xff) + " ");
        }
        log("]");
        buf.position(cur);
    }

    void printchar(CharBuffer buf, int pos) {
        int cur = buf.position();
        log(" (" + pos + ")==[");
        for (int i = 0; i < pos; i++) {
            log("(" + i + ")" + hex(buf.get(i)) + " ");
        }
        log("]");
        buf.position(cur);
    }

    private void checkResultsFromUnicode(ConversionCase cc, ByteBuffer source, ByteBuffer target) {
 
        int len = target.position();
        source.rewind();
        target.rewind();
        
        // remove any BOM signature before checking
        detectUnicodeSignature(target);

        // test to see if the conversion matches actual results
        // remove any BOM signature before checking
        detectUnicodeSignature(target);
        
        len = len-target.position();
        
        if (len != source.remaining()) {
            errln("Test failed: output does not match expected\n"); 
            logln("["+ cc.caseNr + "]:"+cc.charset+"\noutput=" );
            printbytes(target, len); 
            return;
        }
        for (int i = 0; i < source.remaining(); i++) {
            if (target.get() != source.get()) {
                errln("Test failed: output does not match expected\n"); 
                logln("["+ cc.caseNr + "]:"+cc.charset+"\noutput=" );
                printbytes(target, len); 
                return;
            }
        }
        logln("["+ cc.caseNr + "]:"+cc.charset);
        log("output=" ); 
        printbytes(target, len);  
        logln("\nPassed\n");
        return;
    }

    private void checkResultsToUnicode(ConversionCase cc, String source, CharBuffer target) {
 
        int len = target.position(); 
        target.rewind(); 

        // test to see if the conversion matches actual results
        if (len != source.length()) {
            errln("Test failed: output does not match expected\n"); 
            logln("["+ cc.caseNr + "]:"+cc.charset+"\noutput=" );
            printchar(target,len); 
            return;
        }
        for (int i = 0; i < source.length(); i++) {
            if ( ! (hex(target.get(i)).equals(hex(source.charAt(i)))) ) {
                errln("Test failed: output does not match expected\n"); 
                logln("["+ cc.caseNr + "]:"+cc.charset+"\noutput=" );
                printchar(target,len); 
                return;
            }
        }
        logln("["+ cc.caseNr + "]:"+cc.charset);
        log("output=" ); 
         printchar(target,len);  
        logln("\nPassed\n"); 
        return;
    }
}