svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.csv / src / main / java / org / gvsig / fmap / dal / store / csv / simplereaders / CSVReaderSuperCSV.java @ 46338
History | View | Annotate | Download (8.73 KB)
1 | 41617 | jjdelcerro | package org.gvsig.fmap.dal.store.csv.simplereaders; |
---|---|---|---|
2 | |||
3 | 45727 | jjdelcerro | import java.io.File; |
4 | 41617 | jjdelcerro | import java.io.IOException; |
5 | 46054 | omartinez | import java.io.Reader; |
6 | 41617 | jjdelcerro | import java.util.List; |
7 | 45727 | jjdelcerro | import org.apache.commons.io.FilenameUtils; |
8 | 46161 | jjdelcerro | import org.apache.commons.text.StringEscapeUtils; |
9 | 41617 | jjdelcerro | import org.apache.commons.lang3.StringUtils; |
10 | import org.gvsig.fmap.dal.store.csv.CSVStoreParameters; |
||
11 | 45727 | jjdelcerro | import org.gvsig.fmap.dal.store.csv.virtualrows.RandomAccessFileIndex; |
12 | import org.gvsig.fmap.dal.store.csv.virtualrows.RandomAccessFileReader; |
||
13 | import static org.gvsig.fmap.dal.store.csv.virtualrows.RandomAccessFileReader.FILTER_NONE; |
||
14 | import org.gvsig.fmap.dal.store.csv.virtualrows.SuperCSVList; |
||
15 | 41617 | jjdelcerro | import org.gvsig.tools.dynobject.DynObject; |
16 | 45836 | fdiaz | import org.gvsig.tools.task.SimpleTaskStatus; |
17 | 45727 | jjdelcerro | import org.gvsig.tools.util.GetItemWithSize64; |
18 | 41617 | jjdelcerro | import org.slf4j.Logger; |
19 | import org.slf4j.LoggerFactory; |
||
20 | import org.supercsv.comment.CommentStartsWith; |
||
21 | import org.supercsv.io.CsvListReader; |
||
22 | import org.supercsv.prefs.CsvPreference; |
||
23 | import org.supercsv.quote.QuoteMode; |
||
24 | |||
25 | 45727 | jjdelcerro | public class CSVReaderSuperCSV extends AbstractSimpleReader { |
26 | 41617 | jjdelcerro | |
27 | //
|
||
28 | // http://supercsv.sourceforge.net/examples_reading.html
|
||
29 | // http://supercsv.sourceforge.net/apidocs/index.html
|
||
30 | //
|
||
31 | 45685 | jjdelcerro | private static final Logger LOGGER = LoggerFactory.getLogger(CSVReaderSuperCSV.class); |
32 | 41617 | jjdelcerro | |
33 | 42775 | jjdelcerro | private CsvListReader reader;
|
34 | private final CSVStoreParameters parameters; |
||
35 | private List<String> nextLine; |
||
36 | private int columns; |
||
37 | |||
38 | 46054 | omartinez | public CSVReaderSuperCSV(Reader in, CSVStoreParameters parameters) { |
39 | 42775 | jjdelcerro | this(parameters);
|
40 | this.reader = new CsvListReader(in, getCSVPreferences()); |
||
41 | 41617 | jjdelcerro | } |
42 | |||
43 | 45685 | jjdelcerro | public CSVReaderSuperCSV(CSVStoreParameters parameters) {
|
44 | 42775 | jjdelcerro | this.reader = null; |
45 | 41617 | jjdelcerro | this.parameters = parameters;
|
46 | this.reader = null; |
||
47 | 42775 | jjdelcerro | this.nextLine = null; |
48 | this.columns = 0; |
||
49 | 41617 | jjdelcerro | } |
50 | |||
51 | public CSVStoreParameters getParameters() {
|
||
52 | return this.parameters; |
||
53 | } |
||
54 | |||
55 | 42775 | jjdelcerro | @Override
|
56 | 41617 | jjdelcerro | public String[] getHeader() throws IOException { |
57 | return this.reader.getHeader(true); |
||
58 | } |
||
59 | 42775 | jjdelcerro | |
60 | @Override
|
||
61 | public int getColumnsCount() throws IOException { |
||
62 | if( this.columns <= 0 ) { |
||
63 | this.columns = reader.length();
|
||
64 | if( this.columns <= 0 ) { |
||
65 | this.nextLine = this.reader.read(); |
||
66 | this.columns = reader.length();
|
||
67 | } |
||
68 | } |
||
69 | return this.columns; |
||
70 | } |
||
71 | 41617 | jjdelcerro | |
72 | 42775 | jjdelcerro | @Override
|
73 | 45836 | fdiaz | public GetItemWithSize64<List<String>> getVirtualRows(SimpleTaskStatus status) { |
74 | RandomAccessFileReader theReader = null;
|
||
75 | RandomAccessFileIndex theIndex = null;
|
||
76 | try {
|
||
77 | CSVStoreParameters params = getParameters(); |
||
78 | File data_file = CSVStoreParameters.getFile(params);
|
||
79 | if( data_file.length()< 10*1024*1024 ) { |
||
80 | return null; |
||
81 | } |
||
82 | |||
83 | String charset = CSVStoreParameters.getCharset(params);
|
||
84 | File index_file = new File(FilenameUtils.removeExtension(data_file.getAbsolutePath()) + ".idx"); |
||
85 | |||
86 | theReader = new RandomAccessFileReader(data_file, charset);
|
||
87 | theIndex = theReader.createOrOpenIndexOfLines(index_file, false, FILTER_NONE, status);
|
||
88 | |||
89 | SuperCSVList list = new SuperCSVList(
|
||
90 | theReader, |
||
91 | theIndex, |
||
92 | CSVStoreParameters.isFirstLineHeader(getParameters())?1:0 |
||
93 | ); |
||
94 | |||
95 | list.setPreferences(this.getCSVPreferences());
|
||
96 | return list;
|
||
97 | } catch (IOException ex) { |
||
98 | return null; |
||
99 | } finally {
|
||
100 | // We do not close the index or the reader because we need it to remain open
|
||
101 | 45727 | jjdelcerro | // IOUtils.closeQuietly(theReader);
|
102 | // IOUtils.closeQuietly(theIndex);
|
||
103 | 45836 | fdiaz | } |
104 | 45727 | jjdelcerro | } |
105 | |||
106 | @Override
|
||
107 | 41617 | jjdelcerro | public List<String> read() throws IOException { |
108 | 42775 | jjdelcerro | List<String> line; |
109 | if( this.nextLine != null ) { |
||
110 | line = this.nextLine;
|
||
111 | this.nextLine = null; |
||
112 | } else {
|
||
113 | line = this.reader.read();
|
||
114 | } |
||
115 | 45083 | jjdelcerro | if( line!=null ) { |
116 | for (int i = 0; i < line.size(); i++) { |
||
117 | String s = line.get(i);
|
||
118 | if( s!=null ) { |
||
119 | 46161 | jjdelcerro | line.set(i, unescapeCRLF(s)); |
120 | 45083 | jjdelcerro | } |
121 | } |
||
122 | } |
||
123 | 42775 | jjdelcerro | return line;
|
124 | 41617 | jjdelcerro | } |
125 | |||
126 | 42775 | jjdelcerro | @Override
|
127 | 41617 | jjdelcerro | public void close() throws IOException { |
128 | this.reader.close();
|
||
129 | } |
||
130 | |||
131 | 42775 | jjdelcerro | @Override
|
132 | 41617 | jjdelcerro | public List<String> skip(int lines) throws IOException { |
133 | 42775 | jjdelcerro | if( lines <= 0 ) { |
134 | return null; |
||
135 | } |
||
136 | if( this.nextLine != null ) { |
||
137 | this.nextLine = null; |
||
138 | lines--; |
||
139 | } |
||
140 | 41617 | jjdelcerro | List<String> row = null; |
141 | for ( int i = 0; i < lines; i++ ) { |
||
142 | row = reader.read(); |
||
143 | } |
||
144 | return row;
|
||
145 | } |
||
146 | |||
147 | 44667 | jjdelcerro | public final CsvPreference getCSVPreferences() { |
148 | 41617 | jjdelcerro | try {
|
149 | 42775 | jjdelcerro | String s;
|
150 | 41617 | jjdelcerro | char quoteChar;
|
151 | int delimiterChar;
|
||
152 | String endOfLineSymbols;
|
||
153 | |||
154 | DynObject params = this.getParameters();
|
||
155 | |||
156 | 42775 | jjdelcerro | CsvPreference.Builder builder; |
157 | 41617 | jjdelcerro | |
158 | CsvPreference defaultPreference = CSVStoreParameters |
||
159 | .getPredefinedCSVPreferences(params); |
||
160 | if ( defaultPreference == null ) { |
||
161 | defaultPreference = CsvPreference.STANDARD_PREFERENCE; |
||
162 | } |
||
163 | |||
164 | endOfLineSymbols = CSVStoreParameters.getRecordSeparator(params); |
||
165 | if ( StringUtils.isBlank(endOfLineSymbols) ) {
|
||
166 | endOfLineSymbols = defaultPreference.getEndOfLineSymbols(); |
||
167 | } |
||
168 | s = CSVStoreParameters.getQuoteCharacter(params); |
||
169 | if ( StringUtils.isBlank(s) ) {
|
||
170 | quoteChar = (char) defaultPreference.getQuoteChar();
|
||
171 | } else {
|
||
172 | quoteChar = s.charAt(0);
|
||
173 | } |
||
174 | s = CSVStoreParameters.getDelimiter(params); |
||
175 | if ( StringUtils.isBlank(s) ) {
|
||
176 | delimiterChar = defaultPreference.getDelimiterChar(); |
||
177 | } else {
|
||
178 | delimiterChar = s.charAt(0);
|
||
179 | } |
||
180 | |||
181 | builder = new CsvPreference.Builder(quoteChar, delimiterChar,
|
||
182 | endOfLineSymbols); |
||
183 | |||
184 | s = CSVStoreParameters.getCommentStartMarker(params); |
||
185 | if ( !StringUtils.isBlank(s) ) {
|
||
186 | CommentStartsWith cs = new CommentStartsWith(s);
|
||
187 | builder.skipComments(cs); |
||
188 | } |
||
189 | |||
190 | builder.surroundingSpacesNeedQuotes(CSVStoreParameters |
||
191 | .getSurroundingSpacesNeedQuotes(params)); |
||
192 | QuoteMode quoteMode = CSVStoreParameters.getQuoteMode(params); |
||
193 | if ( quoteMode != null ) { |
||
194 | builder.useQuoteMode(quoteMode); |
||
195 | } |
||
196 | return builder.build();
|
||
197 | } catch (Exception e) { |
||
198 | 44667 | jjdelcerro | LOGGER.warn("Can't make preferences for CSV '" + getFullFileName()
|
199 | 41617 | jjdelcerro | + "'.", e);
|
200 | return null; |
||
201 | } |
||
202 | } |
||
203 | |||
204 | private String getFullFileName() { |
||
205 | // Usar solo para mostrar mensajes en el logger.
|
||
206 | 42775 | jjdelcerro | String s;
|
207 | 41617 | jjdelcerro | try {
|
208 | s = getParameters().getFile().getAbsolutePath(); |
||
209 | } catch (Exception e2) { |
||
210 | s = "(unknow)";
|
||
211 | } |
||
212 | return s;
|
||
213 | } |
||
214 | |||
215 | 42775 | jjdelcerro | @Override
|
216 | 41643 | jjdelcerro | public int getLine() { |
217 | if( this.reader==null ) { |
||
218 | return 0; |
||
219 | } |
||
220 | return this.reader.getLineNumber(); |
||
221 | } |
||
222 | |||
223 | 43983 | jjdelcerro | @Override
|
224 | public List<String> nextRowValues() { |
||
225 | try {
|
||
226 | return this.read(); |
||
227 | } catch (IOException ex) { |
||
228 | throw new RuntimeException(ex); |
||
229 | } |
||
230 | } |
||
231 | |||
232 | 46161 | jjdelcerro | public static String escapeCRLF(String s) { |
233 | if( s==null ) { |
||
234 | return s;
|
||
235 | } |
||
236 | String s1 = s;
|
||
237 | s1 = StringUtils.replace(s1, "\\", "\\\\"); |
||
238 | s1 = StringUtils.replace(s1, "\n", "\\n"); |
||
239 | s1 = StringUtils.replace(s1, "\r", "\\r"); |
||
240 | return s1;
|
||
241 | } |
||
242 | |||
243 | public static String unescapeCRLF(String s) { |
||
244 | if( s==null || s.indexOf('\\')==-1 ) { |
||
245 | return s;
|
||
246 | } |
||
247 | String s1 = s;
|
||
248 | s1 = s.replaceAll("(?:^\\\\n)|(?:([^\\\\])\\\\n)","$1\n"); |
||
249 | s1 = s1.replaceAll("(?:^\\\\r)|(?:([^\\\\])\\\\n)","$1\r"); |
||
250 | s1 = StringUtils.replace(s1, "\\\\", "\\"); |
||
251 | return s1;
|
||
252 | } |
||
253 | |||
254 | |||
255 | public static void main(String[] args) { |
||
256 | String s0 = "\\n{\\n \"ANGULO\":\"0.000\",\\n \"\tEXTO\":\"RAVAL ROIG\\\\r\\\\n - \\\\r\\\\nVIRGEN DEL SOCORRO\",\\n \"LINK_POLIGONO\":\"HVCSGISCODE_ENT_11230100000000001\",\\n \"GEOMETRY\":\"00000000014125fe9b57b4a23441503411cb1c432d\"\\n}"; |
||
257 | System.out.println("#"+s0+"#"); |
||
258 | String s1 = s0.replaceAll("(?:^\\\\n)|(?:([^\\\\])\\\\n)","$1\n"); |
||
259 | System.out.println("#"+s1+"#"); |
||
260 | String s2 = s1.replaceAll("([^\\\\])\\\\r","$1{r}"); |
||
261 | System.out.println("#"+s2+"#"); |
||
262 | String s3 = StringEscapeUtils.unescapeCsv(s0);
|
||
263 | System.out.println("#"+s3+"#"); |
||
264 | } |
||
265 | 41617 | jjdelcerro | } |