Application: gvSIG desktop

/*

 * To change this license header, choose License Headers in Project Properties.

 * To change this template file, choose Tools | Templates

 * and open the template in the editor.

 */

package org.gvsig.fmap.dal.store.csv.virtualrows;

import java.io.BufferedReader;

import java.io.File;

import java.io.IOException;

import java.nio.charset.Charset;

import java.util.Date;

import java.util.List;

import java.util.function.Function;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.io.input.CloseShieldReader;

import org.apache.commons.lang3.ArrayUtils;

import static org.gvsig.fmap.dal.store.csv.virtualrows.RandomAccessFileReader.FILTER_NONE;

import org.gvsig.tools.ToolsLocator;

import org.gvsig.tools.library.impl.DefaultLibrariesInitializer;

import org.gvsig.tools.observer.Observable;

import org.gvsig.tools.task.SimpleTaskStatus;

import org.gvsig.tools.task.TaskStatus;

import org.gvsig.tools.task.TaskStatusManager;

import org.supercsv.io.CsvListReader;

import org.supercsv.prefs.CsvPreference;

/**

 *

 * @author jjdelcerro

 */

public class SuperCSVList extends AbstractCSVList {

    private CsvPreference preferences;

    public SuperCSVList(File text, File index, Charset charset) throws IOException {

        super(text, index, charset);

        this.preferences = CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE;

    }

    public SuperCSVList(RandomAccessFileReader reader, RandomAccessFileIndex index, int skipLines) throws IOException {

        super(reader, index);

        this.preferences = CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE;

        this.skipLines  = skipLines;

    }

    public void setPreferences(CsvPreference preferences) {

        this.preferences = preferences;

    }

    @Override

    protected List<String> nextRecord() throws IOException {

        CsvListReader parser = null;

        try {

            CloseShieldReader theReader = CloseShieldReader.wrap(this.reader);

            parser = new CsvListReader(theReader, this.preferences);

            List<String> values = parser.read();

            return values;

        } finally {

            IOUtils.closeQuietly(parser);

        }

    }

    public static void main(String[] args) throws Exception {

        new DefaultLibrariesInitializer().fullInitialize();

        String fname;

        fname = "/home/fdiaz/Descargas/error_union_tablas/Expedientes_CON_REFCAT_trimmed.csv";

//        fname = "/home/fdiaz/Descargas/origen_coordenadas.csv";

//        fname = "/home/jjdelcerro/Descargas/test/origen_coordenadas.csv";

//        fname = "/home/jjdelcerro/Descargas/test/esp_poblaciones.csv";

//        fname = "/home/jjdelcerro/Descargas/test/esp_provincias.csv";

//        fname = "/home/jjdelcerro/Descargas/test/sigpac.csv";

        File data_file = new File(fname);

        File index_file = new File(FilenameUtils.removeExtension(data_file.getAbsolutePath()) + ".idx");

        final TaskStatusManager taskStatusManager = ToolsLocator.getTaskStatusManager();

        taskStatusManager.addObserver((Observable observable, Object notification) -> {

            TaskStatus status = taskStatusManager.getRunningTaskStatusMostRecent();

//            System.out.print("\033[?25l\r");

//            if( status!=null && status.isRunning() ) {

//                System.out.print("\033[?25l\r");

//                System.out.print(status.getTitle()+ " - " + status.getLabel());

//                System.out.print("\033[K\033[?12l\033[?25h");

//            }

//            System.out.flush();

        });

        SimpleTaskStatus status = taskStatusManager.createDefaultSimpleTaskStatus(data_file.getName());

        status.add();

        Function<BufferedReader, Integer> numberOfLinesInRecord = (BufferedReader breader) -> {

            CloseShieldReader theReader = CloseShieldReader.wrap(breader);

            CsvListReader parser = new CsvListReader(theReader, CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE);

            try {

                List<String> values = parser.read();

            } catch (IOException ex) {

                return 1;

            }

            return parser.getLineNumber();

        };

        RandomAccessFileReader reader = new RandomAccessFileReader(data_file, "UTF-8");

        RandomAccessFileIndex index_lines = reader.createIndexOfLines(index_file, false, FILTER_NONE, null, numberOfLinesInRecord);

        CSVList csv = new SuperCSVList(reader, index_lines, 0);

        System.out.println("Lines " + csv.size());

        int n = 0;

        for (List<String> record : csv) {

            if( n<100 ) {

                System.out.println(String.format("%6d: %s", n, ArrayUtils.toString(record)));

            } else if( n==100 ) {

                System.out.println("More records...");

            }

            n++;

        }

        System.out.println("-----------------");

        long t1 = new Date().getTime();

        for (int i = csv.size() - 1; i >= 0; i--) {

            List<String> record = csv.get(i);

            if( i<100 ) {

                System.out.println(String.format("%6d: %s", i, ArrayUtils.toString(record)));

            } else if( i==100 ) {

                System.out.println("More records...");

            }

        }

        long t2 = new Date().getTime();

        System.out.println("Time: " + (t2 - t1) + " ms");

        IOUtils.closeQuietly(csv);

    }

}