svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.csv / src / main / java / org / gvsig / fmap / dal / store / simplereader / virtualrows / RandomAccessFileReader.java @ 47652
History | View | Annotate | Download (37.5 KB)
1 | 47638 | jjdelcerro | package org.gvsig.fmap.dal.store.simplereader.virtualrows; |
---|---|---|---|
2 | 45727 | jjdelcerro | |
3 | import java.io.BufferedReader; |
||
4 | import java.io.File; |
||
5 | import java.io.IOException; |
||
6 | import java.io.RandomAccessFile; |
||
7 | import java.io.Reader; |
||
8 | 45836 | fdiaz | import java.io.UncheckedIOException; |
9 | 45727 | jjdelcerro | import java.nio.ByteBuffer; |
10 | import java.nio.CharBuffer; |
||
11 | import java.nio.channels.Channels; |
||
12 | import java.nio.charset.Charset; |
||
13 | 45836 | fdiaz | import java.util.Iterator; |
14 | import java.util.NoSuchElementException; |
||
15 | import java.util.Spliterator; |
||
16 | import java.util.Spliterators; |
||
17 | 47506 | fdiaz | import java.util.function.Function; |
18 | 45727 | jjdelcerro | import java.util.function.Predicate; |
19 | 45836 | fdiaz | import java.util.stream.Stream; |
20 | import java.util.stream.StreamSupport; |
||
21 | 45727 | jjdelcerro | import org.apache.commons.io.FilenameUtils; |
22 | import org.apache.commons.io.IOUtils; |
||
23 | 47506 | fdiaz | import org.apache.commons.lang3.StringUtils; |
24 | 47652 | fdiaz | import org.apache.commons.lang3.mutable.MutableInt; |
25 | 45727 | jjdelcerro | import org.gvsig.tools.ToolsLocator; |
26 | 45929 | jjdelcerro | import org.gvsig.tools.i18n.I18nManager; |
27 | 45727 | jjdelcerro | import org.gvsig.tools.library.impl.DefaultLibrariesInitializer; |
28 | import org.gvsig.tools.observer.Observable; |
||
29 | import org.gvsig.tools.task.SimpleTaskStatus; |
||
30 | import org.gvsig.tools.task.TaskStatus; |
||
31 | import org.gvsig.tools.task.TaskStatusManager; |
||
32 | |||
33 | /**
|
||
34 | *
|
||
35 | * @author gvSIG Team
|
||
36 | */
|
||
37 | public class RandomAccessFileReader extends Reader { |
||
38 | |||
39 | public static final Predicate<String> FILTER_NONE = (String t) -> false; |
||
40 | |||
41 | protected static final int INDEX_HEADER_FILESIZE = 0; |
||
42 | protected static final int INDEX_HEADER_INDEXCREATIONCOST = 1; |
||
43 | 47506 | fdiaz | |
44 | protected static final int MAX_BUFFER_FOR_LINE = 50*1024; //50K |
||
45 | 45836 | fdiaz | |
46 | 45727 | jjdelcerro | protected RandomAccessFile raf; |
47 | protected Reader reader; |
||
48 | protected long currentPosition; |
||
49 | protected final Charset charset; |
||
50 | protected long lastModified; |
||
51 | |||
52 | public RandomAccessFileReader(File f, String charsetName) throws IOException { |
||
53 | 45836 | fdiaz | this(new RandomAccessFile(f, "r"), Charset.forName(charsetName)); |
54 | 45727 | jjdelcerro | this.lastModified = f.lastModified();
|
55 | } |
||
56 | |||
57 | public RandomAccessFileReader(File f, Charset charset) throws IOException { |
||
58 | 45836 | fdiaz | this(new RandomAccessFile(f, "r"), charset); |
59 | 45727 | jjdelcerro | this.lastModified = f.lastModified();
|
60 | } |
||
61 | |||
62 | public RandomAccessFileReader(RandomAccessFile raf, String charsetName) throws IOException { |
||
63 | this(raf, Charset.forName(charsetName)); |
||
64 | this.lastModified = -1; |
||
65 | } |
||
66 | |||
67 | public RandomAccessFileReader(RandomAccessFile raf, Charset charset) throws IOException { |
||
68 | this.charset = charset;
|
||
69 | this.raf = raf;
|
||
70 | this.reader = null; |
||
71 | this.currentPosition = 0; |
||
72 | this.lastModified = -1; |
||
73 | } |
||
74 | |||
75 | 45948 | jjdelcerro | public Charset getCharset() { |
76 | return this.charset; |
||
77 | } |
||
78 | |||
79 | 45727 | jjdelcerro | @Override
|
80 | public int read(char[] cbuf, int off, int len) throws IOException { |
||
81 | 45836 | fdiaz | if (this.reader == null) { |
82 | 45727 | jjdelcerro | this.createReader();
|
83 | } |
||
84 | int n = this.reader.read(cbuf, off, len); |
||
85 | 45836 | fdiaz | if (n > 0) { |
86 | 45727 | jjdelcerro | // Update current position (bytes) adding the read characters.
|
87 | 45836 | fdiaz | CharBuffer charBuffer = CharBuffer.wrap(cbuf, off, len); |
88 | 45727 | jjdelcerro | ByteBuffer byteBuffer = this.charset.encode(charBuffer); |
89 | this.currentPosition += byteBuffer.limit();
|
||
90 | } |
||
91 | return n;
|
||
92 | } |
||
93 | |||
94 | protected void createReader() { |
||
95 | this.reader = Channels.newReader(this.raf.getChannel(), this.charset.name()); |
||
96 | } |
||
97 | 45836 | fdiaz | |
98 | 45727 | jjdelcerro | // protected InputStream is;
|
99 | // protected void createReader() {
|
||
100 | // if( this.is==null ) {
|
||
101 | // this.is = new InputStream() {
|
||
102 | // @Override
|
||
103 | // public int read() throws IOException {
|
||
104 | // return raf.read();
|
||
105 | // }
|
||
106 | // };
|
||
107 | // }
|
||
108 | // this.reader = new InputStreamReader(this.is, charset);
|
||
109 | // }
|
||
110 | @Override
|
||
111 | public void close() throws IOException { |
||
112 | // IOUtils.closeQuietly(this.is);
|
||
113 | IOUtils.closeQuietly(this.reader);
|
||
114 | IOUtils.closeQuietly(this.raf);
|
||
115 | } |
||
116 | |||
117 | public long getFilePointer() throws IOException { |
||
118 | return this.raf.getFilePointer(); |
||
119 | } |
||
120 | 45836 | fdiaz | |
121 | 45727 | jjdelcerro | public long getCurrentPosition() { |
122 | return this.currentPosition; |
||
123 | } |
||
124 | |||
125 | public void rewind() throws IOException { |
||
126 | this.raf.seek(0); |
||
127 | this.reader = null; |
||
128 | this.currentPosition = 0; |
||
129 | } |
||
130 | 45836 | fdiaz | |
131 | 45727 | jjdelcerro | public void seek(long position) throws IOException { |
132 | this.raf.seek(position);
|
||
133 | this.reader = null; |
||
134 | this.currentPosition = position;
|
||
135 | } |
||
136 | |||
137 | public String readLine() throws IOException { |
||
138 | StringBuilder buffer = new StringBuilder(); |
||
139 | int c = -1; |
||
140 | boolean eol = false; |
||
141 | |||
142 | while (!eol) {
|
||
143 | switch (c = this.read()) { |
||
144 | case -1: |
||
145 | case '\n': |
||
146 | eol = true;
|
||
147 | break;
|
||
148 | case '\r': |
||
149 | eol = true;
|
||
150 | long cur = raf.getFilePointer();
|
||
151 | if ((raf.read()) != '\n') { |
||
152 | raf.seek(cur); |
||
153 | } |
||
154 | break;
|
||
155 | default:
|
||
156 | 45836 | fdiaz | buffer.append((char) c);
|
157 | 45727 | jjdelcerro | break;
|
158 | } |
||
159 | } |
||
160 | if ((c == -1) && (buffer.length() == 0)) { |
||
161 | return null; |
||
162 | } |
||
163 | return buffer.toString();
|
||
164 | } |
||
165 | 45836 | fdiaz | |
166 | public long countLines(Predicate<String> filter, SimpleTaskStatus status) throws IOException { |
||
167 | 47652 | fdiaz | return countLines(filter, new MutableInt(), status); |
168 | } |
||
169 | |||
170 | public long countLines(Predicate<String> filter, MutableInt maxLineLen, SimpleTaskStatus status) throws IOException { |
||
171 | 45836 | fdiaz | if (raf.length() == 0) { |
172 | 45727 | jjdelcerro | return 0; |
173 | } |
||
174 | long savedpos = this.getCurrentPosition(); |
||
175 | long count = -1; |
||
176 | 45836 | fdiaz | if (status != null) { |
177 | 45929 | jjdelcerro | I18nManager i18n = ToolsLocator.getI18nManager(); |
178 | status.message(i18n.getTranslation("_Calculating_number_of_lines"));
|
||
179 | 45727 | jjdelcerro | status.setIndeterminate(); |
180 | } |
||
181 | 47652 | fdiaz | maxLineLen.setValue(0);
|
182 | 47506 | fdiaz | BufferedReader breader = new BufferedReader(this, MAX_BUFFER_FOR_LINE); |
183 | 45727 | jjdelcerro | try {
|
184 | String line;
|
||
185 | 45836 | fdiaz | count = 0;
|
186 | while ((line = breader.readLine()) != null) { |
||
187 | if (status != null) { |
||
188 | if (status.isCancellationRequested()) {
|
||
189 | 45727 | jjdelcerro | return -1; |
190 | } |
||
191 | 45836 | fdiaz | // status.incrementCurrentValue();
|
192 | if((count % 1000) == 0){ |
||
193 | status.setCurValue(count); |
||
194 | } |
||
195 | 45727 | jjdelcerro | } |
196 | 45836 | fdiaz | if (filter.test(line)) {
|
197 | 45727 | jjdelcerro | continue;
|
198 | } |
||
199 | count++; |
||
200 | 47652 | fdiaz | int l = line.getBytes(this.getCharset()).length; |
201 | if(l>maxLineLen.getValue()){
|
||
202 | maxLineLen.setValue(l); |
||
203 | } |
||
204 | 45727 | jjdelcerro | } |
205 | 45836 | fdiaz | if (status != null) { |
206 | 47506 | fdiaz | status.setCurValue(count); |
207 | 45727 | jjdelcerro | status.message("");
|
208 | status.setIndeterminate(); |
||
209 | } |
||
210 | } finally {
|
||
211 | 45836 | fdiaz | this.seek(savedpos);
|
212 | 45727 | jjdelcerro | } |
213 | return count;
|
||
214 | } |
||
215 | 45836 | fdiaz | |
216 | 45727 | jjdelcerro | public boolean isRecomemendedTheRecreationOfTheLinesIndex(File index) { |
217 | RandomAccessFileIndex line_idx = null;
|
||
218 | try {
|
||
219 | 45836 | fdiaz | if (this.lastModified > 0 && this.lastModified > index.lastModified()) { |
220 | 45727 | jjdelcerro | return true; |
221 | } |
||
222 | line_idx = new RandomAccessFileIndex();
|
||
223 | line_idx.open(index); |
||
224 | 45836 | fdiaz | if (this.raf.length() != line_idx.getHeader(INDEX_HEADER_FILESIZE)) { |
225 | 45727 | jjdelcerro | return true; |
226 | } |
||
227 | long creationCost = line_idx.getHeader(INDEX_HEADER_FILESIZE);
|
||
228 | 45836 | fdiaz | if (creationCost < 2000) { // < 2 sec. |
229 | 45727 | jjdelcerro | return true; |
230 | } |
||
231 | 45836 | fdiaz | if (line_idx.get(-1) == 0) { |
232 | 45727 | jjdelcerro | // if last index == 0, index is corrupt
|
233 | return true; |
||
234 | } |
||
235 | // FIXME: isValidIndexOfLines not full implemented
|
||
236 | // Podria comprobarse que una muestra de 4 o 5 bloques de datos
|
||
237 | // repartidas por el fichero tengan el checksum correcto
|
||
238 | return false; |
||
239 | } catch (IOException ex) { |
||
240 | return true; |
||
241 | } finally {
|
||
242 | IOUtils.closeQuietly(line_idx); |
||
243 | } |
||
244 | } |
||
245 | |||
246 | 45836 | fdiaz | public RandomAccessFileIndex createOrOpenIndexOfLines(File index, Predicate<String> filter, SimpleTaskStatus status) throws IOException { |
247 | 45727 | jjdelcerro | return this.createOrOpenIndexOfLines(index, false, filter, status); |
248 | } |
||
249 | |||
250 | 45836 | fdiaz | public RandomAccessFileIndex createOrOpenIndexOfLines(File index, boolean safe, Predicate<String> filter, SimpleTaskStatus status) throws IOException { |
251 | 47506 | fdiaz | return createOrOpenIndexOfLines(index, safe, filter, status, null); |
252 | } |
||
253 | |||
254 | public RandomAccessFileIndex createOrOpenIndexOfLines(File index, boolean safe, Predicate<String> filter, SimpleTaskStatus status, Function<BufferedReader,Integer> numberOfLines) throws IOException { |
||
255 | 45836 | fdiaz | if (this.isRecomemendedTheRecreationOfTheLinesIndex(index)) { |
256 | 47506 | fdiaz | return this.createIndexOfLines(index, safe, filter, status, numberOfLines); |
257 | 45727 | jjdelcerro | } |
258 | return new RandomAccessFileIndex(index); |
||
259 | } |
||
260 | |||
261 | 45836 | fdiaz | public RandomAccessFileIndex createIndexOfLines(File index, Predicate<String> filter, SimpleTaskStatus status) throws IOException { |
262 | 45727 | jjdelcerro | return this.createIndexOfLines(index, false, filter, status); |
263 | } |
||
264 | 45836 | fdiaz | |
265 | public RandomAccessFileIndex createIndexOfLines(File index, boolean safe, Predicate<String> filter, SimpleTaskStatus status) throws IOException { |
||
266 | 47506 | fdiaz | return createIndexOfLines(index, safe, filter, status, null); |
267 | } |
||
268 | |||
269 | public RandomAccessFileIndex createIndexOfLines(File index, boolean safe, Predicate<String> filter, SimpleTaskStatus status, Function<BufferedReader,Integer> numberOfLines) throws IOException { |
||
270 | 47652 | fdiaz | MutableInt maxLineLen = new MutableInt();
|
271 | long countLines = this.countLines(filter, maxLineLen, status); |
||
272 | 45836 | fdiaz | if (countLines < 1) { |
273 | 45727 | jjdelcerro | return null; |
274 | } |
||
275 | 47652 | fdiaz | int maxBufferForLine = maxLineLen.getValue()+1024; |
276 | 45836 | fdiaz | RandomAccessFileIndex line_idx = new RandomAccessFileIndex();
|
277 | 45727 | jjdelcerro | line_idx.create(index, countLines); |
278 | 45836 | fdiaz | |
279 | 45727 | jjdelcerro | long savedpos = this.getCurrentPosition(); |
280 | try {
|
||
281 | 45836 | fdiaz | if (status != null) { |
282 | 45929 | jjdelcerro | I18nManager i18n = ToolsLocator.getI18nManager(); |
283 | 45836 | fdiaz | status.push(); |
284 | 45929 | jjdelcerro | status.message(i18n.getTranslation("_Creating_the_index_of_the_lines"));
|
285 | 45727 | jjdelcerro | status.setRangeOfValues(0, line_idx.size64());
|
286 | status.setCurValue(0);
|
||
287 | } |
||
288 | long t1 = System.currentTimeMillis(); |
||
289 | 47506 | fdiaz | String line = null; |
290 | 45727 | jjdelcerro | int lineno = 0; |
291 | long position = 0; |
||
292 | 45836 | fdiaz | // line_idx.set(lineno++, position);
|
293 | if (safe) {
|
||
294 | 45727 | jjdelcerro | // Don't use buffered reader, slow and safe calculate position
|
295 | 45836 | fdiaz | int x = (int) (countLines / 100); |
296 | while (lineno < countLines) { //true ) { |
||
297 | 45727 | jjdelcerro | line = this.readLine();
|
298 | 45836 | fdiaz | if (line == null) { |
299 | 45727 | jjdelcerro | break;
|
300 | } |
||
301 | 45836 | fdiaz | if (filter.test(line)) {
|
302 | 45727 | jjdelcerro | continue;
|
303 | } |
||
304 | 45836 | fdiaz | line_idx.set(lineno++, position); |
305 | if (status != null) { |
||
306 | if (status.isCancellationRequested()) {
|
||
307 | 45727 | jjdelcerro | status.cancel(); |
308 | return null; |
||
309 | } |
||
310 | 45836 | fdiaz | // status.incrementCurrentValue();
|
311 | if((lineno % x) == 0){ |
||
312 | status.setCurValue(lineno); |
||
313 | } |
||
314 | 45727 | jjdelcerro | } |
315 | position = this.getCurrentPosition();
|
||
316 | 45836 | fdiaz | // line_idx.set(lineno++, position);
|
317 | 45727 | jjdelcerro | } |
318 | 45836 | fdiaz | status.setCurValue(lineno); |
319 | 45727 | jjdelcerro | } else {
|
320 | // Use buffered reader, fast and unsafe calculate position.
|
||
321 | 47506 | fdiaz | StringBuilder builder = new StringBuilder(); |
322 | 47652 | fdiaz | MyBufferedReader breader = new MyBufferedReader(this, maxBufferForLine); |
323 | 45836 | fdiaz | while (lineno < countLines) {
|
324 | 45727 | jjdelcerro | this.seek(position);
|
325 | 47506 | fdiaz | breader.clean(); |
326 | if(numberOfLines == null){ |
||
327 | line = breader.readLine(); |
||
328 | } else {
|
||
329 | 47652 | fdiaz | breader.mark(maxBufferForLine); |
330 | 47506 | fdiaz | Integer nextLine = numberOfLines.apply(breader);
|
331 | breader.reset(); |
||
332 | builder.setLength(0);
|
||
333 | for (int i = 0; i < nextLine; i++) { |
||
334 | String l = breader.readLine();
|
||
335 | if(l != null){ |
||
336 | builder.append(l); |
||
337 | } else {
|
||
338 | break;
|
||
339 | } |
||
340 | } |
||
341 | line = StringUtils.defaultIfBlank(builder.toString(), null);
|
||
342 | } |
||
343 | 45836 | fdiaz | if (line == null) { |
344 | 45727 | jjdelcerro | break;
|
345 | } |
||
346 | 45836 | fdiaz | if (filter.test(line)) {
|
347 | 45727 | jjdelcerro | continue;
|
348 | } |
||
349 | 45836 | fdiaz | line_idx.set(lineno++, position); |
350 | if (status != null) { |
||
351 | if (status.isCancellationRequested()) {
|
||
352 | 45727 | jjdelcerro | status.cancel(); |
353 | return null; |
||
354 | } |
||
355 | status.incrementCurrentValue(); |
||
356 | } |
||
357 | 45836 | fdiaz | CharBuffer charBuffer = null; |
358 | 45727 | jjdelcerro | // ? Y si hay un \r\n ?
|
359 | 45836 | fdiaz | if(breader.isSkipLf()){
|
360 | charBuffer = CharBuffer.wrap(line + "\r\n"); |
||
361 | } else {
|
||
362 | charBuffer = CharBuffer.wrap(line + "\n"); |
||
363 | } |
||
364 | ByteBuffer byteBuffer = this.charset.encode(charBuffer); |
||
365 | 45727 | jjdelcerro | position += byteBuffer.limit(); |
366 | |||
367 | 45836 | fdiaz | // line_idx.set(lineno++, position);
|
368 | 45727 | jjdelcerro | } |
369 | } |
||
370 | long t2 = System.currentTimeMillis(); |
||
371 | 47506 | fdiaz | line_idx.setNumElements(lineno); |
372 | 45727 | jjdelcerro | line_idx.setHeader(INDEX_HEADER_FILESIZE, this.raf.length());
|
373 | 45836 | fdiaz | line_idx.setHeader(INDEX_HEADER_INDEXCREATIONCOST, t2 - t1); |
374 | if (status != null) { |
||
375 | 45727 | jjdelcerro | status.message("");
|
376 | 45836 | fdiaz | status.pop(); |
377 | 45727 | jjdelcerro | } |
378 | return line_idx;
|
||
379 | } finally {
|
||
380 | 45836 | fdiaz | this.seek(savedpos);
|
381 | 45727 | jjdelcerro | } |
382 | } |
||
383 | 45836 | fdiaz | |
384 | 45727 | jjdelcerro | public long getLastLinesIndexCreationCost(RandomAccessFileIndex index) { |
385 | return index.getHeader(INDEX_HEADER_INDEXCREATIONCOST);
|
||
386 | } |
||
387 | 45836 | fdiaz | |
388 | 45727 | jjdelcerro | public static void main(String[] args) throws Exception { |
389 | new DefaultLibrariesInitializer().fullInitialize();
|
||
390 | 45836 | fdiaz | |
391 | 45727 | jjdelcerro | String fname;
|
392 | fname = "/home/jjdelcerro/Descargas/test/origen_coordenadas.csv";
|
||
393 | // fname = "/home/jjdelcerro/Descargas/test/esp_poblaciones.csv";
|
||
394 | // fname = "/home/jjdelcerro/Descargas/test/esp_provincias.csv";
|
||
395 | // fname = "/home/jjdelcerro/Descargas/test/sigpac.csv";
|
||
396 | |||
397 | File data_file = new File(fname); |
||
398 | File idx_file = new File(FilenameUtils.removeExtension(data_file.getAbsolutePath()) + ".idx"); |
||
399 | |||
400 | final TaskStatusManager taskStatusManager = ToolsLocator.getTaskStatusManager();
|
||
401 | taskStatusManager.addObserver((Observable observable, Object notification) -> { |
||
402 | TaskStatus status = taskStatusManager.getRunningTaskStatusMostRecent(); |
||
403 | // System.out.print("\033[?25l\r");
|
||
404 | // if( status!=null && status.isRunning() ) {
|
||
405 | // System.out.print("\033[?25l\r");
|
||
406 | // System.out.print(status.getTitle()+ " - " + status.getLabel());
|
||
407 | // System.out.print("\033[K\033[?12l\033[?25h");
|
||
408 | // }
|
||
409 | // System.out.flush();
|
||
410 | }); |
||
411 | SimpleTaskStatus status = taskStatusManager.createDefaultSimpleTaskStatus(data_file.getName()); |
||
412 | status.add(); |
||
413 | 45836 | fdiaz | |
414 | RandomAccessFileReader reader = new RandomAccessFileReader(data_file, "UTF-8"); |
||
415 | System.out.println("Index '" + idx_file.getName() + "', is creation recomended: " + reader.isRecomemendedTheRecreationOfTheLinesIndex(idx_file)); |
||
416 | 45727 | jjdelcerro | RandomAccessFileIndex lines_idx = reader.createOrOpenIndexOfLines(idx_file, FILTER_NONE, status); |
417 | 45836 | fdiaz | |
418 | 45727 | jjdelcerro | for (int linenumber = 0; linenumber < lines_idx.size(); linenumber++) { |
419 | long lineoffset = lines_idx.get(linenumber);
|
||
420 | reader.seek(lineoffset); |
||
421 | 47506 | fdiaz | MyBufferedReader breader = new MyBufferedReader(reader, MAX_BUFFER_FOR_LINE);
|
422 | 45727 | jjdelcerro | String line = breader.readLine();
|
423 | 45836 | fdiaz | if (linenumber < 100) { |
424 | System.out.println(String.format("%6d/%d: %s", lineoffset, linenumber, line)); |
||
425 | } else if (linenumber == 100) { |
||
426 | 45727 | jjdelcerro | System.out.println("More records..."); |
427 | } |
||
428 | } |
||
429 | 45836 | fdiaz | |
430 | 45727 | jjdelcerro | System.out.println("------------------------------------"); |
431 | |||
432 | 45836 | fdiaz | for (int linenumber = lines_idx.size() - 1; linenumber >= 0; linenumber--) { |
433 | 45727 | jjdelcerro | long lineoffset = lines_idx.get(linenumber);
|
434 | reader.seek(lineoffset); |
||
435 | 47506 | fdiaz | MyBufferedReader breader = new MyBufferedReader(reader, MAX_BUFFER_FOR_LINE);
|
436 | 45727 | jjdelcerro | String line = breader.readLine();
|
437 | 45836 | fdiaz | if (linenumber < 100) { |
438 | System.out.println(String.format("%6d/%d: %s", lineoffset, linenumber, line)); |
||
439 | } else if (linenumber == 100) { |
||
440 | 45727 | jjdelcerro | System.out.println("More records..."); |
441 | } |
||
442 | } |
||
443 | 45836 | fdiaz | |
444 | } |
||
445 | |||
446 | 47506 | fdiaz | /*
|
447 | Copy of java's BufferedReader adding clean and isSkipLf methods
|
||
448 | */
|
||
449 | 45836 | fdiaz | public static class MyBufferedReader extends BufferedReader { |
450 | |||
451 | private Reader in; |
||
452 | |||
453 | private char cb[]; |
||
454 | private int nChars, nextChar; |
||
455 | |||
456 | private static final int INVALIDATED = -2; |
||
457 | private static final int UNMARKED = -1; |
||
458 | private int markedChar = UNMARKED; |
||
459 | private int readAheadLimit = 0; |
||
460 | /* Valid only when markedChar > 0 */
|
||
461 | |||
462 | /**
|
||
463 | * If the next character is a line feed, skip it
|
||
464 | */
|
||
465 | private boolean skipLF = false; |
||
466 | |||
467 | /**
|
||
468 | * The skipLF flag when the mark was set
|
||
469 | */
|
||
470 | private boolean markedSkipLF = false; |
||
471 | |||
472 | private static int defaultCharBufferSize = 8192; |
||
473 | private static int defaultExpectedLineLength = 80; |
||
474 | |||
475 | /**
|
||
476 | * Creates a buffering character-input stream that uses an input buffer
|
||
477 | * of the specified size.
|
||
478 | *
|
||
479 | * @param in A Reader
|
||
480 | * @param sz Input-buffer size
|
||
481 | *
|
||
482 | * @exception IllegalArgumentException If {@code sz <= 0}
|
||
483 | */
|
||
484 | public MyBufferedReader(Reader in, int sz) { |
||
485 | super(in);
|
||
486 | if (sz <= 0) { |
||
487 | throw new IllegalArgumentException("Buffer size <= 0"); |
||
488 | } |
||
489 | this.in = in;
|
||
490 | cb = new char[sz]; |
||
491 | nextChar = nChars = 0;
|
||
492 | } |
||
493 | |||
494 | /**
|
||
495 | * Creates a buffering character-input stream that uses a default-sized
|
||
496 | * input buffer.
|
||
497 | *
|
||
498 | * @param in A Reader
|
||
499 | */
|
||
500 | public MyBufferedReader(Reader in) { |
||
501 | this(in, defaultCharBufferSize);
|
||
502 | } |
||
503 | 47506 | fdiaz | |
504 | 45836 | fdiaz | /**
|
505 | * Checks to make sure that the stream has not been closed
|
||
506 | */
|
||
507 | private void ensureOpen() throws IOException { |
||
508 | if (in == null) { |
||
509 | throw new IOException("Stream closed"); |
||
510 | } |
||
511 | } |
||
512 | |||
513 | /**
|
||
514 | * Fills the input buffer, taking the mark into account if it is valid.
|
||
515 | */
|
||
516 | private void fill() throws IOException { |
||
517 | int dst;
|
||
518 | if (markedChar <= UNMARKED) {
|
||
519 | /* No mark */
|
||
520 | dst = 0;
|
||
521 | } else {
|
||
522 | /* Marked */
|
||
523 | int delta = nextChar - markedChar;
|
||
524 | if (delta >= readAheadLimit) {
|
||
525 | /* Gone past read-ahead limit: Invalidate mark */
|
||
526 | markedChar = INVALIDATED; |
||
527 | readAheadLimit = 0;
|
||
528 | dst = 0;
|
||
529 | } else {
|
||
530 | if (readAheadLimit <= cb.length) {
|
||
531 | /* Shuffle in the current buffer */
|
||
532 | System.arraycopy(cb, markedChar, cb, 0, delta); |
||
533 | markedChar = 0;
|
||
534 | dst = delta; |
||
535 | } else {
|
||
536 | /* Reallocate buffer to accommodate read-ahead limit */
|
||
537 | char ncb[] = new char[readAheadLimit]; |
||
538 | System.arraycopy(cb, markedChar, ncb, 0, delta); |
||
539 | cb = ncb; |
||
540 | markedChar = 0;
|
||
541 | dst = delta; |
||
542 | } |
||
543 | nextChar = nChars = delta; |
||
544 | } |
||
545 | } |
||
546 | |||
547 | int n;
|
||
548 | do {
|
||
549 | n = in.read(cb, dst, cb.length - dst); |
||
550 | } while (n == 0); |
||
551 | if (n > 0) { |
||
552 | nChars = dst + n; |
||
553 | nextChar = dst; |
||
554 | } |
||
555 | } |
||
556 | |||
557 | /**
|
||
558 | * Reads a single character.
|
||
559 | *
|
||
560 | * @return The character read, as an integer in the range 0 to 65535
|
||
561 | * (<tt>0x00-0xffff</tt>), or -1 if the end of the stream has been
|
||
562 | * reached
|
||
563 | * @exception IOException If an I/O error occurs
|
||
564 | */
|
||
565 | 47506 | fdiaz | @Override
|
566 | 45836 | fdiaz | public int read() throws IOException { |
567 | synchronized (lock) {
|
||
568 | ensureOpen(); |
||
569 | for (;;) {
|
||
570 | if (nextChar >= nChars) {
|
||
571 | fill(); |
||
572 | if (nextChar >= nChars) {
|
||
573 | return -1; |
||
574 | } |
||
575 | } |
||
576 | if (skipLF) {
|
||
577 | skipLF = false;
|
||
578 | if (cb[nextChar] == '\n') { |
||
579 | nextChar++; |
||
580 | continue;
|
||
581 | } |
||
582 | } |
||
583 | return cb[nextChar++];
|
||
584 | } |
||
585 | } |
||
586 | } |
||
587 | |||
588 | /**
|
||
589 | * Reads characters into a portion of an array, reading from the
|
||
590 | * underlying stream if necessary.
|
||
591 | */
|
||
592 | private int read1(char[] cbuf, int off, int len) throws IOException { |
||
593 | if (nextChar >= nChars) {
|
||
594 | /* If the requested length is at least as large as the buffer, and
|
||
595 | if there is no mark/reset activity, and if line feeds are not
|
||
596 | being skipped, do not bother to copy the characters into the
|
||
597 | local buffer. In this way buffered streams will cascade
|
||
598 | harmlessly. */
|
||
599 | if (len >= cb.length && markedChar <= UNMARKED && !skipLF) {
|
||
600 | return in.read(cbuf, off, len);
|
||
601 | } |
||
602 | fill(); |
||
603 | } |
||
604 | if (nextChar >= nChars) {
|
||
605 | return -1; |
||
606 | } |
||
607 | if (skipLF) {
|
||
608 | skipLF = false;
|
||
609 | if (cb[nextChar] == '\n') { |
||
610 | nextChar++; |
||
611 | if (nextChar >= nChars) {
|
||
612 | fill(); |
||
613 | } |
||
614 | if (nextChar >= nChars) {
|
||
615 | return -1; |
||
616 | } |
||
617 | } |
||
618 | } |
||
619 | int n = Math.min(len, nChars - nextChar); |
||
620 | System.arraycopy(cb, nextChar, cbuf, off, n);
|
||
621 | nextChar += n; |
||
622 | return n;
|
||
623 | } |
||
624 | |||
625 | /**
|
||
626 | * Reads characters into a portion of an array.
|
||
627 | *
|
||
628 | * <p>
|
||
629 | * This method implements the general contract of the corresponding
|
||
630 | * <code>{@link Reader#read(char[], int, int) read}</code> method of the
|
||
631 | * <code>{@link Reader}</code> class. As an additional convenience, it
|
||
632 | * attempts to read as many characters as possible by repeatedly
|
||
633 | * invoking the <code>read</code> method of the underlying stream. This
|
||
634 | * iterated <code>read</code> continues until one of the following
|
||
635 | * conditions becomes true: <ul>
|
||
636 | *
|
||
637 | * <li> The specified number of characters have been read,
|
||
638 | *
|
||
639 | * <li> The <code>read</code> method of the underlying stream returns
|
||
640 | * <code>-1</code>, indicating end-of-file, or
|
||
641 | *
|
||
642 | * <li> The <code>ready</code> method of the underlying stream returns
|
||
643 | * <code>false</code>, indicating that further input requests would
|
||
644 | * block.
|
||
645 | *
|
||
646 | * </ul> If the first <code>read</code> on the underlying stream returns
|
||
647 | * <code>-1</code> to indicate end-of-file then this method returns
|
||
648 | * <code>-1</code>. Otherwise this method returns the number of
|
||
649 | * characters actually read.
|
||
650 | *
|
||
651 | * <p>
|
||
652 | * Subclasses of this class are encouraged, but not required, to attempt
|
||
653 | * to read as many characters as possible in the same fashion.
|
||
654 | *
|
||
655 | * <p>
|
||
656 | * Ordinarily this method takes characters from this stream's character
|
||
657 | * buffer, filling it from the underlying stream as necessary. If,
|
||
658 | * however, the buffer is empty, the mark is not valid, and the
|
||
659 | * requested length is at least as large as the buffer, then this method
|
||
660 | * will read characters directly from the underlying stream into the
|
||
661 | * given array. Thus redundant <code>BufferedReader</code>s will not
|
||
662 | * copy data unnecessarily.
|
||
663 | *
|
||
664 | * @param cbuf Destination buffer
|
||
665 | * @param off Offset at which to start storing characters
|
||
666 | * @param len Maximum number of characters to read
|
||
667 | *
|
||
668 | * @return The number of characters read, or -1 if the end of the stream
|
||
669 | * has been reached
|
||
670 | *
|
||
671 | * @exception IOException If an I/O error occurs
|
||
672 | */
|
||
673 | 47506 | fdiaz | @Override
|
674 | 45836 | fdiaz | public int read(char cbuf[], int off, int len) throws IOException { |
675 | synchronized (lock) {
|
||
676 | ensureOpen(); |
||
677 | if ((off < 0) || (off > cbuf.length) || (len < 0) |
||
678 | || ((off + len) > cbuf.length) || ((off + len) < 0)) {
|
||
679 | throw new IndexOutOfBoundsException(); |
||
680 | } else if (len == 0) { |
||
681 | return 0; |
||
682 | } |
||
683 | |||
684 | int n = read1(cbuf, off, len);
|
||
685 | if (n <= 0) { |
||
686 | return n;
|
||
687 | } |
||
688 | while ((n < len) && in.ready()) {
|
||
689 | int n1 = read1(cbuf, off + n, len - n);
|
||
690 | if (n1 <= 0) { |
||
691 | break;
|
||
692 | } |
||
693 | n += n1; |
||
694 | } |
||
695 | return n;
|
||
696 | } |
||
697 | } |
||
698 | |||
699 | /**
|
||
700 | * Reads a line of text. A line is considered to be terminated by any
|
||
701 | * one of a line feed ('\n'), a carriage return ('\r'), or a carriage
|
||
702 | * return followed immediately by a linefeed.
|
||
703 | *
|
||
704 | * @param ignoreLF If true, the next '\n' will be skipped
|
||
705 | *
|
||
706 | * @return A String containing the contents of the line, not including
|
||
707 | * any line-termination characters, or null if the end of the stream has
|
||
708 | * been reached
|
||
709 | *
|
||
710 | * @see java.io.LineNumberReader#readLine()
|
||
711 | *
|
||
712 | * @exception IOException If an I/O error occurs
|
||
713 | */
|
||
714 | String readLine(boolean ignoreLF) throws IOException { |
||
715 | StringBuilder s = null; |
||
716 | int startChar;
|
||
717 | |||
718 | synchronized (lock) {
|
||
719 | ensureOpen(); |
||
720 | boolean omitLF = ignoreLF || skipLF;
|
||
721 | |||
722 | bufferLoop: |
||
723 | for (;;) {
|
||
724 | |||
725 | if (nextChar >= nChars) {
|
||
726 | fill(); |
||
727 | } |
||
728 | if (nextChar >= nChars) {
|
||
729 | /* EOF */
|
||
730 | if (s != null && s.length() > 0) { |
||
731 | return s.toString();
|
||
732 | } else {
|
||
733 | return null; |
||
734 | } |
||
735 | } |
||
736 | boolean eol = false; |
||
737 | char c = 0; |
||
738 | int i;
|
||
739 | |||
740 | /* Skip a leftover '\n', if necessary */
|
||
741 | if (omitLF && (cb[nextChar] == '\n')) { |
||
742 | nextChar++; |
||
743 | } |
||
744 | skipLF = false;
|
||
745 | omitLF = false;
|
||
746 | |||
747 | charLoop: |
||
748 | for (i = nextChar; i < nChars; i++) {
|
||
749 | c = cb[i]; |
||
750 | if ((c == '\n') || (c == '\r')) { |
||
751 | eol = true;
|
||
752 | break charLoop;
|
||
753 | } |
||
754 | } |
||
755 | |||
756 | startChar = nextChar; |
||
757 | nextChar = i; |
||
758 | |||
759 | if (eol) {
|
||
760 | String str;
|
||
761 | if (s == null) { |
||
762 | str = new String(cb, startChar, i - startChar); |
||
763 | } else {
|
||
764 | s.append(cb, startChar, i - startChar); |
||
765 | str = s.toString(); |
||
766 | } |
||
767 | nextChar++; |
||
768 | if (c == '\r') { |
||
769 | skipLF = true;
|
||
770 | } |
||
771 | return str;
|
||
772 | } |
||
773 | |||
774 | if (s == null) { |
||
775 | s = new StringBuilder(defaultExpectedLineLength); |
||
776 | } |
||
777 | s.append(cb, startChar, i - startChar); |
||
778 | } |
||
779 | } |
||
780 | } |
||
781 | |||
782 | /**
|
||
783 | * Reads a line of text. A line is considered to be terminated by any
|
||
784 | * one of a line feed ('\n'), a carriage return ('\r'), or a carriage
|
||
785 | * return followed immediately by a linefeed.
|
||
786 | *
|
||
787 | * @return A String containing the contents of the line, not including
|
||
788 | * any line-termination characters, or null if the end of the stream has
|
||
789 | * been reached
|
||
790 | *
|
||
791 | * @exception IOException If an I/O error occurs
|
||
792 | *
|
||
793 | * @see java.nio.file.Files#readAllLines
|
||
794 | */
|
||
795 | 47506 | fdiaz | @Override
|
796 | 45836 | fdiaz | public String readLine() throws IOException { |
797 | return readLine(false); |
||
798 | } |
||
799 | |||
800 | /**
|
||
801 | * Skips characters.
|
||
802 | *
|
||
803 | * @param n The number of characters to skip
|
||
804 | *
|
||
805 | * @return The number of characters actually skipped
|
||
806 | *
|
||
807 | * @exception IllegalArgumentException If <code>n</code> is negative.
|
||
808 | * @exception IOException If an I/O error occurs
|
||
809 | */
|
||
810 | 47506 | fdiaz | @Override
|
811 | 45836 | fdiaz | public long skip(long n) throws IOException { |
812 | if (n < 0L) { |
||
813 | throw new IllegalArgumentException("skip value is negative"); |
||
814 | } |
||
815 | synchronized (lock) {
|
||
816 | ensureOpen(); |
||
817 | long r = n;
|
||
818 | while (r > 0) { |
||
819 | if (nextChar >= nChars) {
|
||
820 | fill(); |
||
821 | } |
||
822 | if (nextChar >= nChars) /* EOF */ { |
||
823 | break;
|
||
824 | } |
||
825 | if (skipLF) {
|
||
826 | skipLF = false;
|
||
827 | if (cb[nextChar] == '\n') { |
||
828 | nextChar++; |
||
829 | } |
||
830 | } |
||
831 | long d = nChars - nextChar;
|
||
832 | if (r <= d) {
|
||
833 | nextChar += r; |
||
834 | r = 0;
|
||
835 | break;
|
||
836 | } else {
|
||
837 | r -= d; |
||
838 | nextChar = nChars; |
||
839 | } |
||
840 | } |
||
841 | return n - r;
|
||
842 | } |
||
843 | } |
||
844 | |||
845 | /**
|
||
846 | * Tells whether this stream is ready to be read. A buffered character
|
||
847 | * stream is ready if the buffer is not empty, or if the underlying
|
||
848 | * character stream is ready.
|
||
849 | *
|
||
850 | * @exception IOException If an I/O error occurs
|
||
851 | */
|
||
852 | 47506 | fdiaz | @Override
|
853 | 45836 | fdiaz | public boolean ready() throws IOException { |
854 | synchronized (lock) {
|
||
855 | ensureOpen(); |
||
856 | |||
857 | /*
|
||
858 | * If newline needs to be skipped and the next char to be read
|
||
859 | * is a newline character, then just skip it right away.
|
||
860 | */
|
||
861 | if (skipLF) {
|
||
862 | /* Note that in.ready() will return true if and only if the next
|
||
863 | * read on the stream will not block.
|
||
864 | */
|
||
865 | if (nextChar >= nChars && in.ready()) {
|
||
866 | fill(); |
||
867 | } |
||
868 | if (nextChar < nChars) {
|
||
869 | if (cb[nextChar] == '\n') { |
||
870 | nextChar++; |
||
871 | } |
||
872 | skipLF = false;
|
||
873 | } |
||
874 | } |
||
875 | return (nextChar < nChars) || in.ready();
|
||
876 | } |
||
877 | } |
||
878 | |||
879 | /**
|
||
880 | * Tells whether this stream supports the mark() operation, which it
|
||
881 | * does.
|
||
882 | */
|
||
883 | 47506 | fdiaz | @Override
|
884 | 45836 | fdiaz | public boolean markSupported() { |
885 | return true; |
||
886 | } |
||
887 | |||
888 | /**
|
||
889 | * Marks the present position in the stream. Subsequent calls to reset()
|
||
890 | * will attempt to reposition the stream to this point.
|
||
891 | *
|
||
892 | * @param readAheadLimit Limit on the number of characters that may be
|
||
893 | * read while still preserving the mark. An attempt to reset the stream
|
||
894 | * after reading characters up to this limit or beyond may fail. A limit
|
||
895 | * value larger than the size of the input buffer will cause a new
|
||
896 | * buffer to be allocated whose size is no smaller than limit. Therefore
|
||
897 | * large values should be used with care.
|
||
898 | *
|
||
899 | * @exception IllegalArgumentException If {@code readAheadLimit < 0}
|
||
900 | * @exception IOException If an I/O error occurs
|
||
901 | */
|
||
902 | 47506 | fdiaz | @Override
|
903 | 45836 | fdiaz | public void mark(int readAheadLimit) throws IOException { |
904 | if (readAheadLimit < 0) { |
||
905 | throw new IllegalArgumentException("Read-ahead limit < 0"); |
||
906 | } |
||
907 | synchronized (lock) {
|
||
908 | ensureOpen(); |
||
909 | this.readAheadLimit = readAheadLimit;
|
||
910 | markedChar = nextChar; |
||
911 | markedSkipLF = skipLF; |
||
912 | } |
||
913 | } |
||
914 | |||
915 | /**
|
||
916 | * Resets the stream to the most recent mark.
|
||
917 | *
|
||
918 | * @exception IOException If the stream has never been marked, or if the
|
||
919 | * mark has been invalidated
|
||
920 | */
|
||
921 | 47506 | fdiaz | @Override
|
922 | 45836 | fdiaz | public void reset() throws IOException { |
923 | synchronized (lock) {
|
||
924 | ensureOpen(); |
||
925 | if (markedChar < 0) { |
||
926 | throw new IOException((markedChar == INVALIDATED) |
||
927 | ? "Mark invalid"
|
||
928 | : "Stream not marked");
|
||
929 | } |
||
930 | nextChar = markedChar; |
||
931 | skipLF = markedSkipLF; |
||
932 | } |
||
933 | } |
||
934 | |||
935 | 47506 | fdiaz | @Override
|
936 | 45836 | fdiaz | public void close() throws IOException { |
937 | synchronized (lock) {
|
||
938 | if (in == null) { |
||
939 | return;
|
||
940 | } |
||
941 | try {
|
||
942 | in.close(); |
||
943 | } finally {
|
||
944 | in = null;
|
||
945 | cb = null;
|
||
946 | } |
||
947 | } |
||
948 | } |
||
949 | |||
950 | /**
|
||
951 | * Returns a {@code Stream}, the elements of which are lines read from
|
||
952 | * this {@code BufferedReader}. The {@link Stream} is lazily populated,
|
||
953 | * i.e., read only occurs during the
|
||
954 | * <a href="../util/stream/package-summary.html#StreamOps">terminal
|
||
955 | * stream operation</a>.
|
||
956 | *
|
||
957 | * <p>
|
||
958 | * The reader must not be operated on during the execution of the
|
||
959 | * terminal stream operation. Otherwise, the result of the terminal
|
||
960 | * stream operation is undefined.
|
||
961 | *
|
||
962 | * <p>
|
||
963 | * After execution of the terminal stream operation there are no
|
||
964 | * guarantees that the reader will be at a specific position from which
|
||
965 | * to read the next character or line.
|
||
966 | *
|
||
967 | * <p>
|
||
968 | * If an {@link IOException} is thrown when accessing the underlying
|
||
969 | * {@code BufferedReader}, it is wrapped in an {@link
|
||
970 | * UncheckedIOException} which will be thrown from the {@code Stream}
|
||
971 | * method that caused the read to take place. This method will return a
|
||
972 | * Stream if invoked on a BufferedReader that is closed. Any operation
|
||
973 | * on that stream that requires reading from the BufferedReader after it
|
||
974 | * is closed, will cause an UncheckedIOException to be thrown.
|
||
975 | *
|
||
976 | * @return a {@code Stream<String>} providing the lines of text
|
||
977 | * described by this {@code BufferedReader}
|
||
978 | *
|
||
979 | * @since 1.8
|
||
980 | */
|
||
981 | 47506 | fdiaz | @Override
|
982 | 45836 | fdiaz | public Stream<String> lines() { |
983 | Iterator<String> iter = new Iterator<String>() { |
||
984 | String nextLine = null; |
||
985 | |||
986 | @Override
|
||
987 | public boolean hasNext() { |
||
988 | if (nextLine != null) { |
||
989 | return true; |
||
990 | } else {
|
||
991 | try {
|
||
992 | nextLine = readLine(); |
||
993 | return (nextLine != null); |
||
994 | } catch (IOException e) { |
||
995 | throw new UncheckedIOException(e); |
||
996 | } |
||
997 | } |
||
998 | } |
||
999 | |||
1000 | @Override
|
||
1001 | public String next() { |
||
1002 | if (nextLine != null || hasNext()) { |
||
1003 | String line = nextLine;
|
||
1004 | nextLine = null;
|
||
1005 | return line;
|
||
1006 | } else {
|
||
1007 | throw new NoSuchElementException(); |
||
1008 | } |
||
1009 | } |
||
1010 | }; |
||
1011 | return StreamSupport.stream(Spliterators.spliteratorUnknownSize(
|
||
1012 | iter, Spliterator.ORDERED | Spliterator.NONNULL), false);
|
||
1013 | } |
||
1014 | 45727 | jjdelcerro | |
1015 | 45836 | fdiaz | public boolean isSkipLf() { |
1016 | return this.skipLF; |
||
1017 | } |
||
1018 | 47506 | fdiaz | |
1019 | public void clean() { |
||
1020 | nextChar = nChars = 0;
|
||
1021 | markedChar = UNMARKED; |
||
1022 | readAheadLimit = 0;
|
||
1023 | skipLF = false;
|
||
1024 | markedSkipLF = false;
|
||
1025 | |||
1026 | } |
||
1027 | 45727 | jjdelcerro | } |
1028 | 45836 | fdiaz | |
1029 | 45727 | jjdelcerro | } |