Application: gvSIG desktop

/*

 * To change this license header, choose License Headers in Project Properties.

 * To change this template file, choose Tools | Templates

 * and open the template in the editor.

 */

package org.gvsig.fmap.dal.store.csv.csvlib;

import java.io.Closeable;

import java.io.DataOutputStream;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.RandomAccessFile;

import java.nio.LongBuffer;

import java.nio.MappedByteBuffer;

import java.nio.channels.FileChannel.MapMode;

import java.nio.charset.Charset;

import java.util.AbstractList;

import java.util.Arrays;

import java.util.Date;

import java.util.List;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.io.IOUtils;

/**

 *

 * @author jjdelcerro

 */

public class TextFileAsList extends AbstractList<String> implements Closeable {

    private final File text;

    private final File idx;

    private final Charset charset;

    private final Bytes buffer;

    private RandomAccessFile text_raf;

    private RandomAccessFile idx_raf;

    private MappedByteBuffer idx_map;

    private LongBuffer idx_buffer;

    public TextFileAsList(File text, File idx, Charset charset) {

        this.text = text;

        this.idx = idx;

        this.charset = charset;

        this.buffer = new Bytes(this.charset);

    }

    public static void create_index(File csv, File idx, Charset charset) throws IOException {

        FileOutputStream idx_fos = null;

        DataOutputStream idx_dos = null;

        RandomAccessFile csv_raf = null;

        try {

            idx_fos = new FileOutputStream(idx);

            idx_dos = new DataOutputStream(idx_fos);

            csv_raf = new RandomAccessFile(csv, "r");

            Bytes buffer = new Bytes(charset);

            String line;

            long pos = csv_raf.getFilePointer();

            while ((line = readLine(csv_raf, buffer)) != null) {

                idx_dos.writeLong(pos);

                pos = csv_raf.getFilePointer();

            }

        } finally {

            IOUtils.closeQuietly(idx_dos);

            IOUtils.closeQuietly(idx_fos);

            IOUtils.closeQuietly(csv_raf);

        }

    }

    public void open() {

        try {

            text_raf = new RandomAccessFile(this.text, "r");

            idx_raf = new RandomAccessFile(this.idx, "r");

            long idx_size = idx_raf.length();

            idx_map = idx_raf.getChannel().map(MapMode.READ_ONLY, 0, idx_size);

            idx_buffer = idx_map.asLongBuffer();

        } catch (IOException ex) {

            throw new RuntimeException("Can't open csv/idx files.", ex);

        }

    }

    private long getFilePointer(int lineno) {

        if (idx_buffer == null) {

            this.open();

        }

        return idx_buffer.get(lineno);

    }

    @Override

    public void close() {

        IOUtils.closeQuietly(this.text_raf);

        IOUtils.closeQuietly(this.idx_raf);

        this.idx_buffer = null;

        this.idx_map = null;

    }

    @Override

    public String get(int index) {

        if (text_raf == null) {

            this.open();

        }

        try {

            long pos = this.getFilePointer(index);

            this.text_raf.seek(pos);

            return readLine(text_raf, this.buffer);

        } catch (IOException ex) {

            throw new RuntimeException("Can't access to " + index + " element.", ex);

        }

    }

    @Override

    public int size() {

        if (idx_buffer == null) {

            this.open();

        }

        return this.idx_buffer.capacity();

    }

    @Override

    public void clear() {

        throw new UnsupportedOperationException();

    }

    private static class Bytes {

        private static final int DEFAULT_CAPACITY = 1024;

        private final Charset charset;

        private byte[] buffer = new byte[DEFAULT_CAPACITY];

        private int pos = 0;

        public Bytes(Charset charset) {

            this.charset = charset;

        }

        public void reset() {

            this.pos = 0;

        }

        public void append(int value) {

            if (pos >= buffer.length) {

                grow(buffer.length + DEFAULT_CAPACITY);

            }

            this.buffer[pos++] = (byte) value;

        }

        private void grow(int minCapacity) {

            buffer = Arrays.copyOf(buffer, minCapacity);

        }

        public int length() {

            return this.pos;

        }

        @Override

        public String toString() {

            String s = new String(buffer, 0, pos, charset);

            return s;

        }

    }

    private static String readLine(RandomAccessFile raf, Bytes buffer) throws IOException {

        buffer.reset();

        int c = -1;

        boolean eol = false;

        while (!eol) {

            switch (c = raf.read()) {

                case -1:

                case '\n':

                    eol = true;

                    break;

                case '\r':

                    eol = true;

                    long cur = raf.getFilePointer();

                    if ((raf.read()) != '\n') {

                        raf.seek(cur);

                    }

                    break;

                default:

                    buffer.append(c);

                    break;

            }

        }

        if ((c == -1) && (buffer.length() == 0)) {

            return null;

        }

        return buffer.toString();

    }

    public static void main(String[] args) throws Exception {

        String fname;

//        fname = "/home/jjdelcerro/Descargas/test/origen_coordenadas.csv";

//        fname = "/home/jjdelcerro/Descargas/test/esp_poblaciones.csv";

        fname = "/home/jjdelcerro/Descargas/test/esp_provincias.csv";

        File csv_file = new File(fname);

        File idx_file = new File(FilenameUtils.removeExtension(csv_file.getAbsolutePath()) + ".idx");

        TextFileAsList.create_index(csv_file, idx_file, Charset.forName("UTF-8"));

        List<String> text = new TextFileAsList(csv_file, idx_file, Charset.forName("UTF-8"));

        System.out.println("Lines " + text.size());

        for (String line : text) {

//            System.out.println(line);

        }

//        System.out.println("-----------------");

        long t1 = new Date().getTime();

        for (int i = text.size()-1; i >= 0 ; i--) {

            String line = text.get(i);

//            System.out.println(String.format("%6d: %s", i, line));

        }

        long t2 = new Date().getTime();

        System.out.println("Time: "+ (t2-t1) + " ms");

        IOUtils.closeQuietly((Closeable) text);

    }

}