Application: gvSIG desktop

package org.gvsig.fmap.dal.store.csv.simplereaders;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.IOException;

import java.io.InputStreamReader;

import java.io.Reader;

import java.io.UnsupportedEncodingException;

import java.util.List;

import java.util.function.Function;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.io.input.CloseShieldReader;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.text.StringEscapeUtils;

import org.gvsig.fmap.dal.store.csv.CSVStoreParameters;

import org.gvsig.fmap.dal.store.simplereader.virtualrows.RandomAccessFileIndex;

import org.gvsig.fmap.dal.store.simplereader.virtualrows.RandomAccessFileReader;

import static org.gvsig.fmap.dal.store.simplereader.virtualrows.RandomAccessFileReader.FILTER_NONE;

import org.gvsig.fmap.dal.store.csv.virtualrows.SuperCSVList;

import org.gvsig.fmap.dal.store.simplereader.simplereaders.AbstractSimpleReader;

import org.gvsig.tools.dynobject.DynObject;

import org.gvsig.tools.task.SimpleTaskStatus;

import org.gvsig.tools.util.GetItemWithSize64;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.supercsv.comment.CommentStartsWith;

import org.supercsv.io.CsvListReader;

import org.supercsv.prefs.CsvPreference;

import org.supercsv.quote.QuoteMode;

public class CSVReaderSuperCSV extends AbstractSimpleReader {

    //

    // http://supercsv.sourceforge.net/examples_reading.html

    // http://supercsv.sourceforge.net/apidocs/index.html

    //

    private static final Logger LOGGER = LoggerFactory.getLogger(CSVReaderSuperCSV.class);

    private CsvListReader reader;

    private final CSVStoreParameters parameters;

    private List<String>  nextLine;

    private int columns;

    public CSVReaderSuperCSV(Reader in, CSVStoreParameters parameters) {

        this(parameters);

        this.reader = new CsvListReader(in, getCSVPreferences());

    }    

    public CSVReaderSuperCSV(CSVStoreParameters parameters) {

        this.reader = null;

        this.parameters = parameters;

        this.reader = null;

        this.nextLine = null;

        this.columns = 0;

    }

    public CSVStoreParameters getParameters() {

        return this.parameters;

    }

    @Override

    public String[] getHeader() throws IOException {

        return this.reader.getHeader(true);

    }

    @Override

    public int getColumnsCount() throws IOException {

        if( this.columns <= 0 ) {

            this.columns = reader.length();

            if( this.columns <= 0 ) {

                this.nextLine = this.reader.read();

                this.columns = reader.length();

            }

        }

        return this.columns;

    }

    private boolean hasMultilineRecords(SimpleTaskStatus status) throws FileNotFoundException, UnsupportedEncodingException, IOException {

        FileInputStream fis = null;

        InputStreamReader theReader = null;

        BufferedReader breader = null;

        try {

            CSVStoreParameters params = getParameters();

            File data_file = CSVStoreParameters.getFile(params);

            String charset = CSVStoreParameters.getCharset(params);

            fis = new FileInputStream(data_file);

            theReader = new InputStreamReader(fis, charset);

            breader = new BufferedReader(theReader);

            CsvListReader parser = new CsvListReader(breader, getCSVPreferences());

//        int firstRecordLine = 0;

            while (parser.read() != null) {

                if (parser.getLineNumber() != parser.getRowNumber()) {

                    return true;

                }

            }

            return false;

        } finally {

            IOUtils.closeQuietly(breader);

            IOUtils.closeQuietly(theReader);

            IOUtils.closeQuietly(fis);

        }

    }

    @Override

    public GetItemWithSize64<List<String>>  getVirtualRows(SimpleTaskStatus status) {

        RandomAccessFileReader theReader = null;

        RandomAccessFileIndex theIndex = null;

        try {

            CSVStoreParameters params = getParameters();

            File data_file = CSVStoreParameters.getFile(params);

            if( data_file.length()< 10*1024*1024 ) {

                return null;

            }

            Function<BufferedReader, Integer> numberOfLinesInRecord = null;

            if(this.hasMultilineRecords(status)){

                numberOfLinesInRecord = new Function<BufferedReader, Integer>() {

                    @Override

                    public Integer apply(BufferedReader breader) {

                        CloseShieldReader theReader = CloseShieldReader.wrap(breader);

                        CsvListReader parser = new CsvListReader(theReader, getCSVPreferences());

                        try {

                            List<String> values = parser.read();

                        } catch (IOException ex) {

                            return 1;

                        }

                        return parser.getLineNumber();

                    }

                };

            }

            String charset = CSVStoreParameters.getCharset(params);

            File index_file = getIndexFile(data_file);

            theReader = new RandomAccessFileReader(data_file, charset);

            theIndex = theReader.createOrOpenIndexOfLines(index_file, false, FILTER_NONE, status, numberOfLinesInRecord);

            SuperCSVList list = new SuperCSVList(

                    theReader, 

                    theIndex, 

                    CSVStoreParameters.isFirstLineHeader(getParameters())?1:0

            );

            list.setPreferences(this.getCSVPreferences());

            return list;

        } catch (IOException ex) {

            return null;

        } finally {

            // We do not close the index or the reader because we need it to remain open

//            IOUtils.closeQuietly(theReader);

//            IOUtils.closeQuietly(theIndex);

        }

    }

    @Override

    public List<String> read() throws IOException {

        List<String> line;

        if( this.nextLine != null ) {

            line = this.nextLine;

            this.nextLine = null;

        } else {

            line = this.reader.read();

        }

        if( line!=null ) {

            for (int i = 0; i < line.size(); i++) {

                String s = line.get(i);

                if( s!=null ) {

                    line.set(i, unescapeCRLF(s));

                }

            }

        }

        return line;

    }

    @Override

    public void close() throws IOException {

        this.reader.close();

    }

    @Override

    public List<String> skip(int lines) throws IOException {

        if( lines <= 0 ) {

            return null;

        }

        if( this.nextLine != null ) {

            this.nextLine = null;

            lines--;

        }

        List<String> row = null;

        for ( int i = 0; i < lines; i++ ) {

            row = reader.read();

        }

        return row;

    }

    public final CsvPreference getCSVPreferences() {

        try {

            String s;

            char quoteChar;

            int delimiterChar;

            String endOfLineSymbols;

            DynObject params = this.getParameters();

            CsvPreference.Builder builder;

            CsvPreference defaultPreference = CSVStoreParameters

                    .getPredefinedCSVPreferences(params);

            if ( defaultPreference == null ) {

                defaultPreference = CsvPreference.STANDARD_PREFERENCE;

            }

            endOfLineSymbols = CSVStoreParameters.getRecordSeparator(params);

            if ( StringUtils.isBlank(endOfLineSymbols) ) {

                endOfLineSymbols = defaultPreference.getEndOfLineSymbols();

            }

            s = CSVStoreParameters.getQuoteCharacter(params);

            if ( StringUtils.isBlank(s) ) {

                quoteChar = (char) defaultPreference.getQuoteChar();

            } else {

                quoteChar = s.charAt(0);

            }

            s = CSVStoreParameters.getDelimiter(params);

            if ( StringUtils.isBlank(s) ) {

                delimiterChar = defaultPreference.getDelimiterChar();

            } else {

                delimiterChar = s.charAt(0);

            }

            builder = new CsvPreference.Builder(quoteChar, delimiterChar,

                    endOfLineSymbols);

            s = CSVStoreParameters.getCommentStartMarker(params);

            if ( !StringUtils.isBlank(s) ) {

                CommentStartsWith cs = new CommentStartsWith(s);

                builder.skipComments(cs);

            }

            builder.surroundingSpacesNeedQuotes(CSVStoreParameters

                    .getSurroundingSpacesNeedQuotes(params));

            QuoteMode quoteMode = CSVStoreParameters.getQuoteMode(params);

            if ( quoteMode != null ) {

                builder.useQuoteMode(quoteMode);

            }

            return builder.build();

        } catch (Exception e) {

            LOGGER.warn("Can't make preferences for CSV '" + getFullFileName()

                    + "'.", e);

            return null;

        }

    }

    private String getFullFileName() {

        // Usar solo para mostrar mensajes en el logger.

        String s;

        try {

            s = getParameters().getFile().getAbsolutePath();

        } catch (Exception e2) {

            s = "(unknow)";

        }

        return s;        

    }

    @Override

    public int getLine() {

        if( this.reader==null ) {

            return 0;

        }

        return this.reader.getLineNumber();

    }

    @Override

    public List<String> nextRowValues() {

        try {

            return this.read();

        } catch (IOException ex) {

            throw new RuntimeException(ex);

        }

    }

    public static String escapeCRLF(String s) {

        if( s==null ) {

            return s;

        }

        String s1 = s;

        s1 = StringUtils.replace(s1, "\\", "\\\\");

        s1 = StringUtils.replace(s1, "\n", "\\n");

        s1 = StringUtils.replace(s1, "\r", "\\r");

        return s1;

    }

    public static String unescapeCRLF(String s) {

        if( s==null || s.indexOf('\\')==-1 ) {

            return s;

        }

        String s1 = s;

        s1 = s.replaceAll("(?:^\\\\n)|(?:([^\\\\])\\\\n)","$1\n");

        s1 = s1.replaceAll("(?:^\\\\r)|(?:([^\\\\])\\\\n)","$1\r");

        s1 = StringUtils.replace(s1, "\\\\", "\\");

        return s1;

    }

    public static void main(String[] args) {

        String s0 = "\\n{\\n   \"ANGULO\":\"0.000\",\\n    \"\tEXTO\":\"RAVAL ROIG\\\\r\\\\n - \\\\r\\\\nVIRGEN DEL SOCORRO\",\\n    \"LINK_POLIGONO\":\"HVCSGISCODE_ENT_11230100000000001\",\\n    \"GEOMETRY\":\"00000000014125fe9b57b4a23441503411cb1c432d\"\\n}";

        System.out.println("#"+s0+"#");

        String s1 = s0.replaceAll("(?:^\\\\n)|(?:([^\\\\])\\\\n)","$1\n");

        System.out.println("#"+s1+"#");

        String s2 = s1.replaceAll("([^\\\\])\\\\r","$1{r}");

        System.out.println("#"+s2+"#");

        String s3 = StringEscapeUtils.unescapeCsv(s0);

        System.out.println("#"+s3+"#");

    }

    public static File getIndexFile(File data_file) {

        if (data_file == null){

            return null;

        }

        File index_file = new File(FilenameUtils.removeExtension(data_file.getAbsolutePath()) + ".idx");

        return index_file;

    }

}