Revision 47506 trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/virtualrows/RandomAccessFileReader.java

View differences:

RandomAccessFileReader.java
14 14
import java.util.NoSuchElementException;
15 15
import java.util.Spliterator;
16 16
import java.util.Spliterators;
17
import java.util.function.Function;
17 18
import java.util.function.Predicate;
18 19
import java.util.stream.Stream;
19 20
import java.util.stream.StreamSupport;
20 21
import org.apache.commons.io.FilenameUtils;
21 22
import org.apache.commons.io.IOUtils;
23
import org.apache.commons.lang3.StringUtils;
22 24
import org.gvsig.tools.ToolsLocator;
23 25
import org.gvsig.tools.i18n.I18nManager;
24 26
import org.gvsig.tools.library.impl.DefaultLibrariesInitializer;
......
37 39

  
38 40
    protected static final int INDEX_HEADER_FILESIZE = 0;
39 41
    protected static final int INDEX_HEADER_INDEXCREATIONCOST = 1;
42
    
43
    protected static final int MAX_BUFFER_FOR_LINE = 50*1024; //50K
40 44

  
41 45
    protected RandomAccessFile raf;
42 46
    protected Reader reader;
......
169 173
            status.message(i18n.getTranslation("_Calculating_number_of_lines"));
170 174
            status.setIndeterminate();
171 175
        }
172
        BufferedReader breader = new BufferedReader(this, 10240);
176
        BufferedReader breader = new BufferedReader(this, MAX_BUFFER_FOR_LINE);
173 177
        try {
174 178
            String line;
175 179
            count = 0;
......
188 192
                }
189 193
                count++;
190 194
            }
191
            status.setCurValue(count);
192 195
            if (status != null) {
196
                status.setCurValue(count);
193 197
                status.message("");
194 198
                status.setIndeterminate();
195 199
            }
......
234 238
    }
235 239

  
236 240
    public RandomAccessFileIndex createOrOpenIndexOfLines(File index, boolean safe, Predicate<String> filter, SimpleTaskStatus status) throws IOException {
241
        return createOrOpenIndexOfLines(index, safe, filter, status, null);
242
    }
243

  
244
    public RandomAccessFileIndex createOrOpenIndexOfLines(File index, boolean safe, Predicate<String> filter, SimpleTaskStatus status, Function<BufferedReader,Integer> numberOfLines) throws IOException {
237 245
        if (this.isRecomemendedTheRecreationOfTheLinesIndex(index)) {
238
            return this.createIndexOfLines(index, safe, filter, status);
246
            return this.createIndexOfLines(index, safe, filter, status, numberOfLines);
239 247
        }
240 248
        return new RandomAccessFileIndex(index);
241 249
    }
......
245 253
    }
246 254

  
247 255
    public RandomAccessFileIndex createIndexOfLines(File index, boolean safe, Predicate<String> filter, SimpleTaskStatus status) throws IOException {
256
        return createIndexOfLines(index, safe, filter, status, null);
257
    }
258
    
259
    public RandomAccessFileIndex createIndexOfLines(File index, boolean safe, Predicate<String> filter, SimpleTaskStatus status, Function<BufferedReader,Integer> numberOfLines) throws IOException {
248 260
        long countLines = this.countLines(filter, status);
249 261
        if (countLines < 1) {
250 262
            return null;
......
262 274
                status.setCurValue(0);
263 275
            }
264 276
            long t1 = System.currentTimeMillis();
265
            String line;
277
            String line = null;
266 278
            int lineno = 0;
267 279
            long position = 0;
268 280
//            line_idx.set(lineno++, position);
......
294 306
                status.setCurValue(lineno);
295 307
            } else {
296 308
                // Use buffered reader, fast and unsafe calculate position.
309
                StringBuilder builder = new StringBuilder();
310
                MyBufferedReader breader = new MyBufferedReader(this, MAX_BUFFER_FOR_LINE);
297 311
                while (lineno < countLines) {
298 312
                    this.seek(position);
299
                    MyBufferedReader breader = new MyBufferedReader(this, 10240);
300
                    line = breader.readLine();
313
                    breader.clean();
314
                    if(numberOfLines == null){
315
                        line = breader.readLine();
316
                    } else {
317
                        breader.mark(MAX_BUFFER_FOR_LINE);
318
                        Integer nextLine = numberOfLines.apply(breader);
319
                        breader.reset();
320
                        builder.setLength(0);
321
                        for (int i = 0; i < nextLine; i++) {
322
                            String l = breader.readLine();
323
                            if(l != null){
324
                                builder.append(l);
325
                            } else {
326
                                break;
327
                            }
328
                        }
329
                        line = StringUtils.defaultIfBlank(builder.toString(), null);
330
                    }
301 331
                    if (line == null) {
302 332
                        break;
303 333
                    }
......
326 356
                }
327 357
            }
328 358
            long t2 = System.currentTimeMillis();
359
            line_idx.setNumElements(lineno);
329 360
            line_idx.setHeader(INDEX_HEADER_FILESIZE, this.raf.length());
330 361
            line_idx.setHeader(INDEX_HEADER_INDEXCREATIONCOST, t2 - t1);
331 362
            if (status != null) {
......
375 406
        for (int linenumber = 0; linenumber < lines_idx.size(); linenumber++) {
376 407
            long lineoffset = lines_idx.get(linenumber);
377 408
            reader.seek(lineoffset);
378
            MyBufferedReader breader = new MyBufferedReader(reader, 10240);
409
            MyBufferedReader breader = new MyBufferedReader(reader, MAX_BUFFER_FOR_LINE);
379 410
            String line = breader.readLine();
380 411
            if (linenumber < 100) {
381 412
                System.out.println(String.format("%6d/%d: %s", lineoffset, linenumber, line));
......
389 420
        for (int linenumber = lines_idx.size() - 1; linenumber >= 0; linenumber--) {
390 421
            long lineoffset = lines_idx.get(linenumber);
391 422
            reader.seek(lineoffset);
392
            MyBufferedReader breader = new MyBufferedReader(reader, 10240);
423
            MyBufferedReader breader = new MyBufferedReader(reader, MAX_BUFFER_FOR_LINE);
393 424
            String line = breader.readLine();
394 425
            if (linenumber < 100) {
395 426
                System.out.println(String.format("%6d/%d: %s", lineoffset, linenumber, line));
......
400 431

  
401 432
    }
402 433

  
434
    /*
435
        Copy of java's BufferedReader adding clean and isSkipLf methods
436
    */
403 437
    public static class MyBufferedReader extends BufferedReader {
404 438

  
405 439
        private Reader in;
......
454 488
        public MyBufferedReader(Reader in) {
455 489
            this(in, defaultCharBufferSize);
456 490
        }
457

  
491
        
458 492
        /**
459 493
         * Checks to make sure that the stream has not been closed
460 494
         */
......
516 550
         * reached
517 551
         * @exception IOException If an I/O error occurs
518 552
         */
553
        @Override
519 554
        public int read() throws IOException {
520 555
            synchronized (lock) {
521 556
                ensureOpen();
......
623 658
         *
624 659
         * @exception IOException If an I/O error occurs
625 660
         */
661
        @Override
626 662
        public int read(char cbuf[], int off, int len) throws IOException {
627 663
            synchronized (lock) {
628 664
                ensureOpen();
......
744 780
         *
745 781
         * @see java.nio.file.Files#readAllLines
746 782
         */
783
        @Override
747 784
        public String readLine() throws IOException {
748 785
            return readLine(false);
749 786
        }
......
758 795
         * @exception IllegalArgumentException If <code>n</code> is negative.
759 796
         * @exception IOException If an I/O error occurs
760 797
         */
798
        @Override
761 799
        public long skip(long n) throws IOException {
762 800
            if (n < 0L) {
763 801
                throw new IllegalArgumentException("skip value is negative");
......
799 837
         *
800 838
         * @exception IOException If an I/O error occurs
801 839
         */
840
        @Override
802 841
        public boolean ready() throws IOException {
803 842
            synchronized (lock) {
804 843
                ensureOpen();
......
829 868
         * Tells whether this stream supports the mark() operation, which it
830 869
         * does.
831 870
         */
871
        @Override
832 872
        public boolean markSupported() {
833 873
            return true;
834 874
        }
......
847 887
         * @exception IllegalArgumentException If {@code readAheadLimit < 0}
848 888
         * @exception IOException If an I/O error occurs
849 889
         */
890
        @Override
850 891
        public void mark(int readAheadLimit) throws IOException {
851 892
            if (readAheadLimit < 0) {
852 893
                throw new IllegalArgumentException("Read-ahead limit < 0");
......
865 906
         * @exception IOException If the stream has never been marked, or if the
866 907
         * mark has been invalidated
867 908
         */
909
        @Override
868 910
        public void reset() throws IOException {
869 911
            synchronized (lock) {
870 912
                ensureOpen();
......
878 920
            }
879 921
        }
880 922

  
923
        @Override
881 924
        public void close() throws IOException {
882 925
            synchronized (lock) {
883 926
                if (in == null) {
......
923 966
         *
924 967
         * @since 1.8
925 968
         */
969
        @Override
926 970
        public Stream<String> lines() {
927 971
            Iterator<String> iter = new Iterator<String>() {
928 972
                String nextLine = null;
......
959 1003
        public boolean isSkipLf() {
960 1004
            return this.skipLF;
961 1005
        }
962

  
1006
        
1007
        public void clean() {
1008
            nextChar = nChars = 0;
1009
            markedChar = UNMARKED;
1010
            readAheadLimit = 0;
1011
            skipLF = false;
1012
            markedSkipLF = false;
1013
            
1014
        }
963 1015
    }
964 1016

  
965 1017
}

Also available in: Unified diff