svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.lib / src / main / java / org / gvsig / fmap / dal / store / simplereader / SimpleReaderFeatureTypeLoader.java @ 47846
History | View | Annotate | Download (12.7 KB)
1 | 47636 | fdiaz | /*
|
---|---|---|---|
2 | * To change this license header, choose License Headers in Project Properties.
|
||
3 | * To change this template file, choose Tools | Templates
|
||
4 | * and open the template in the editor.
|
||
5 | */
|
||
6 | package org.gvsig.fmap.dal.store.simplereader; |
||
7 | |||
8 | import java.io.IOException; |
||
9 | import java.io.InputStreamReader; |
||
10 | import java.io.Reader; |
||
11 | import java.util.Locale; |
||
12 | import java.util.Map; |
||
13 | import org.apache.commons.io.IOUtils; |
||
14 | import org.apache.commons.lang3.ArrayUtils; |
||
15 | import org.apache.commons.lang3.StringUtils; |
||
16 | import org.gvsig.fmap.dal.DataTypes; |
||
17 | import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor; |
||
18 | import org.gvsig.fmap.dal.feature.EditableFeatureType; |
||
19 | 47638 | jjdelcerro | import org.gvsig.fmap.dal.store.simplereader.simplereaders.SimpleReader; |
20 | 47636 | fdiaz | import org.gvsig.tools.dynobject.Tags; |
21 | import org.gvsig.tools.task.SimpleTaskStatus; |
||
22 | import org.slf4j.Logger; |
||
23 | import org.slf4j.LoggerFactory; |
||
24 | |||
25 | /**
|
||
26 | *
|
||
27 | * @author fdiaz
|
||
28 | */
|
||
29 | public abstract class SimpleReaderFeatureTypeLoader { |
||
30 | |||
31 | private static final Logger LOGGER = LoggerFactory.getLogger(SimpleReaderFeatureTypeLoader.class); |
||
32 | |||
33 | protected boolean all_fields_declare_type; |
||
34 | 47846 | fdiaz | protected SimpleReaderStoreParameters parameters;
|
35 | 47636 | fdiaz | |
36 | public SimpleReaderFeatureTypeLoader(SimpleReaderStoreParameters parameters) {
|
||
37 | this.parameters = parameters;
|
||
38 | } |
||
39 | |||
40 | 47846 | fdiaz | protected SimpleReaderFeatureTypeLoader() {
|
41 | this.parameters = null; |
||
42 | } |
||
43 | |||
44 | 47636 | fdiaz | protected abstract String getProviderName(); |
45 | protected abstract SimpleReader getSimpleReader(Reader in) throws IOException; |
||
46 | |||
47 | protected FieldTypeParser[] getFieldTypes(String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) { |
||
48 | String fullFileName = parameters.getFile()==null? "":parameters.getFile().getAbsolutePath(); |
||
49 | FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length]; |
||
50 | |||
51 | for (int i = 0; i < fieldTypes.length; i++) { |
||
52 | fieldTypes[i] = new FieldTypeParser(getProviderName(), fullFileName);
|
||
53 | } |
||
54 | |||
55 | if (automaticTypes != null) { |
||
56 | for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) { |
||
57 | fieldTypes[i].detectedValue = automaticTypes[i]; |
||
58 | fieldTypes[i].type = automaticTypes[i].getType(); |
||
59 | } |
||
60 | } |
||
61 | |||
62 | this.all_fields_declare_type = true; |
||
63 | for (int i = 0; i < fieldTypes.length; i++) { |
||
64 | if (!fieldTypes[i].parse(headers[i])) {
|
||
65 | LOGGER.warn("Can't parse header of field "+i+ "( "+headers[i]+") in '"+getProviderName()+"' file '" + fullFileName + "'."); |
||
66 | } |
||
67 | if( fieldTypes[i].type == DataTypes.UNKNOWN ) {
|
||
68 | all_fields_declare_type = false;
|
||
69 | fieldTypes[i].type = DataTypes.STRING; |
||
70 | } |
||
71 | } |
||
72 | |||
73 | String param_types_def = SimpleReaderStoreParameters.getRawFieldTypes(parameters);
|
||
74 | if (StringUtils.isNotBlank(param_types_def)) {
|
||
75 | String sep = SimpleReaderStoreParameters.getDelimiter(param_types_def);
|
||
76 | if (StringUtils.isNotBlank(sep)) {
|
||
77 | String[] param_types = param_types_def.split(sep); |
||
78 | FieldTypeParser parser = new FieldTypeParser(getProviderName(), fullFileName);
|
||
79 | for (String param_type : param_types) { |
||
80 | parser.clear(); |
||
81 | parser.parse(param_type); |
||
82 | for (FieldTypeParser fieldType : fieldTypes) {
|
||
83 | if (StringUtils.equalsIgnoreCase(fieldType.name, parser.name)) {
|
||
84 | fieldType.copyFrom(parser); |
||
85 | break;
|
||
86 | } |
||
87 | } |
||
88 | } |
||
89 | } |
||
90 | } |
||
91 | |||
92 | return fieldTypes;
|
||
93 | } |
||
94 | |||
95 | protected void fillFeatureType(EditableFeatureType fType, FieldTypeParser[] fieldTypes) { |
||
96 | String fullFileName = parameters.getFile()==null? "":parameters.getFile().getAbsolutePath(); |
||
97 | |||
98 | Tags ftypeTags = fType.getTags(); |
||
99 | for (FieldTypeParser fieldType : fieldTypes) {
|
||
100 | EditableFeatureAttributeDescriptor fad = fType.add(fieldType.name, fieldType.type); |
||
101 | if( fieldType.detectedValue!=null ) { |
||
102 | fad.setDisplaySize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));
|
||
103 | fad.setSize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));
|
||
104 | if( fad.getPrecision()<fieldType.detectedValue.getPrecision() ) {
|
||
105 | fad.setPrecision(fieldType.detectedValue.getPrecision()); |
||
106 | } |
||
107 | if( fad.getScale()<fieldType.detectedValue.getScale()) {
|
||
108 | fad.setScale(fieldType.detectedValue.getScale()); |
||
109 | } |
||
110 | } else {
|
||
111 | fad.setDisplaySize(fieldType.size); |
||
112 | } |
||
113 | if (fieldType.type == DataTypes.GEOMETRY ) {
|
||
114 | fad.setGeometryType(fieldType.geomType, fieldType.geomSubtype); |
||
115 | if( fType.getDefaultGeometryAttributeName() == null ) { |
||
116 | fType.setDefaultGeometryAttributeName(fieldType.name); |
||
117 | } |
||
118 | } |
||
119 | Locale locale = null; |
||
120 | if (fieldType.type == DataTypes.TIMESTAMP ) {
|
||
121 | if(!SimpleReaderStoreParameters.isBlankOrDefaultLocale(parameters)){
|
||
122 | locale = SimpleReaderStoreParameters.getLocale(parameters); |
||
123 | } |
||
124 | } else {
|
||
125 | locale = SimpleReaderStoreParameters.getLocale(parameters); |
||
126 | } |
||
127 | fad.setLocale(locale); |
||
128 | for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) { |
||
129 | try {
|
||
130 | switch(entry.getKey().toLowerCase()) {
|
||
131 | case "expression": |
||
132 | // Los campos calculados los procesamos en una segunda
|
||
133 | // pasada, cuando ya estan definidos el resto de los campos
|
||
134 | // ya que pueden requerir campos que aun no se han definido.
|
||
135 | break;
|
||
136 | default:
|
||
137 | fad.set(entry.getKey(), entry.getValue()); |
||
138 | } |
||
139 | } catch (Exception ex) { |
||
140 | LOGGER.warn("Can't set property '"+entry.getKey()+"' of '"+fad.getName()+"'.", ex); |
||
141 | } |
||
142 | } |
||
143 | Tags tags = fad.getTags(); |
||
144 | for (Map.Entry<String, String> entry : fieldType.tags.entrySet()) { |
||
145 | tags.set(entry.getKey(), entry.getValue()); |
||
146 | } |
||
147 | for (Map.Entry<String, String> entry : fieldType.typetags.entrySet()) { |
||
148 | ftypeTags.set(entry.getKey(), entry.getValue()); |
||
149 | } |
||
150 | for (Map.Entry<String, String> entry : fieldType.typeAssignments.entrySet()) { |
||
151 | try {
|
||
152 | fType.set(entry.getKey(), entry.getValue()); |
||
153 | } catch(Exception ex) { |
||
154 | LOGGER.warn("Can't set attribute '"+entry.getKey()+"' in the feature type.", ex); |
||
155 | } |
||
156 | } |
||
157 | } |
||
158 | |||
159 | // Processamos ahora los campos calculados
|
||
160 | for (FieldTypeParser fieldType : fieldTypes) {
|
||
161 | EditableFeatureAttributeDescriptor fad = fType.getEditableAttributeDescriptor(fieldType.name); |
||
162 | for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) { |
||
163 | try {
|
||
164 | switch(entry.getKey().toLowerCase()) {
|
||
165 | case "expression": |
||
166 | fad.set(entry.getKey(), entry.getValue()); |
||
167 | break;
|
||
168 | } |
||
169 | } catch (Exception ex) { |
||
170 | LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+fullFileName+"'.", ex); |
||
171 | } |
||
172 | } |
||
173 | } |
||
174 | } |
||
175 | |||
176 | 47652 | fdiaz | public boolean loadFeatureType(EditableFeatureType featureType, SimpleTaskStatus status) throws IOException { |
177 | return loadFeatureType(featureType, true, status); |
||
178 | } |
||
179 | |||
180 | 47636 | fdiaz | public boolean loadFeatureType(EditableFeatureType featureType, boolean detectTypes, SimpleTaskStatus status) throws IOException { |
181 | InputStreamReader in = null; |
||
182 | SimpleReader reader = null;
|
||
183 | try {
|
||
184 | in = SimpleReaderUtils.openFile( |
||
185 | parameters.getFile(), |
||
186 | SimpleReaderStoreParameters.getCharset(parameters) |
||
187 | ); |
||
188 | |||
189 | reader = getSimpleReader(in); |
||
190 | |||
191 | String[] headers = getHeaders(reader); |
||
192 | |||
193 | AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = null; |
||
194 | if( detectTypes ) {
|
||
195 | detectedTypes = automaticDetectionOfTypes(headers, status); |
||
196 | } |
||
197 | if( StringUtils.isBlank(headers[headers.length-1]) && |
||
198 | (detectedTypes==null || detectedTypes[headers.length-1].isBlank()) ) { |
||
199 | headers = ArrayUtils.remove(headers, headers.length-1);
|
||
200 | 47789 | fdiaz | if(detectedTypes != null) { |
201 | detectedTypes = ArrayUtils.remove(detectedTypes, detectedTypes.length-1);
|
||
202 | } |
||
203 | 47636 | fdiaz | } |
204 | if (detectedTypes != null && detectedTypes.length > headers.length) { |
||
205 | // Se han detectado mas columnas que las que hay en la cabezera,
|
||
206 | // a?adimos mas columnas a la cabezera.
|
||
207 | String[] headers2 = new String[detectedTypes.length]; |
||
208 | for (int i = 0; i < headers2.length; i++) { |
||
209 | if (i < headers.length) {
|
||
210 | headers2[i] = headers[i]; |
||
211 | } else {
|
||
212 | headers2[i] = getFixedHeader(i); |
||
213 | } |
||
214 | } |
||
215 | headers = headers2; |
||
216 | } |
||
217 | for (int i = 0; i < headers.length; i++) { |
||
218 | if (StringUtils.isBlank(headers[i])) {
|
||
219 | headers[i] = getFixedHeader(i); |
||
220 | } |
||
221 | } |
||
222 | // Initialize the feature types
|
||
223 | fillFeatureType(featureType, headers, detectedTypes); |
||
224 | return this.all_fields_declare_type; |
||
225 | } finally {
|
||
226 | IOUtils.closeQuietly(in); |
||
227 | IOUtils.closeQuietly(reader); |
||
228 | } |
||
229 | } |
||
230 | |||
231 | protected String[] getHeaders(SimpleReader reader) throws RuntimeException, IOException { |
||
232 | String headers[]; |
||
233 | headers = SimpleReaderStoreParameters.getHeaders(parameters); |
||
234 | return headers;
|
||
235 | } |
||
236 | |||
237 | protected String[] getFixedHeaders(int count) { |
||
238 | String[] headers = new String[count]; |
||
239 | for (int i = 0; i < headers.length; i++) { |
||
240 | headers[i] = getFixedHeader(i); |
||
241 | } |
||
242 | return headers;
|
||
243 | } |
||
244 | |||
245 | |||
246 | protected String getFixedHeader(int column) { |
||
247 | char[] header = new char[3]; |
||
248 | |||
249 | String s = String.format("%03d", column); |
||
250 | header[0] = (char) (s.charAt(0) + 17); |
||
251 | header[1] = (char) (s.charAt(1) + 17); |
||
252 | header[2] = (char) (s.charAt(2) + 17); |
||
253 | return String.valueOf(header); |
||
254 | } |
||
255 | |||
256 | private AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(String[] headers, SimpleTaskStatus status) throws IOException { |
||
257 | String fullFileName = parameters.getFile()==null? "NULL":parameters.getFile().getAbsolutePath(); |
||
258 | boolean automatic_types_detection = SimpleReaderStoreParameters.getAutomaticTypesDetection(parameters);
|
||
259 | if (!automatic_types_detection) {
|
||
260 | return null; |
||
261 | } |
||
262 | AutomaticDetectionOfTypes.DetectedValue[] types = null; |
||
263 | |||
264 | Reader in = null; |
||
265 | SimpleReader reader = null;
|
||
266 | |||
267 | try {
|
||
268 | in = SimpleReaderUtils.openFile( |
||
269 | parameters.getFile(), |
||
270 | SimpleReaderStoreParameters.getCharset(parameters) |
||
271 | ); |
||
272 | reader = getSimpleReader(in); |
||
273 | AutomaticDetectionOfTypes x = new AutomaticDetectionOfTypes(
|
||
274 | fullFileName |
||
275 | ); |
||
276 | types = x.detect( |
||
277 | headers.length, |
||
278 | reader, |
||
279 | isFirstLineHeader(), |
||
280 | SimpleReaderStoreParameters.getLocale(parameters), |
||
281 | status |
||
282 | ); |
||
283 | } catch (Exception ex) { |
||
284 | int lineno = 0; |
||
285 | if (reader != null) { |
||
286 | lineno = reader.getLine(); |
||
287 | } |
||
288 | throw new RuntimeException("Problems reading file '" + fullFileName + "' near line " + lineno + ".", ex); |
||
289 | |||
290 | } finally {
|
||
291 | IOUtils.closeQuietly(reader); |
||
292 | IOUtils.closeQuietly(in); |
||
293 | } |
||
294 | return types;
|
||
295 | } |
||
296 | |||
297 | protected boolean isFirstLineHeader() { |
||
298 | return false; |
||
299 | } |
||
300 | |||
301 | 47638 | jjdelcerro | protected void fillFeatureType(EditableFeatureType fType, String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) { |
302 | 47636 | fdiaz | fType.setHasOID(true);
|
303 | FieldTypeParser[] fieldTypes = getFieldTypes(headers, automaticTypes);
|
||
304 | fillFeatureType(fType, fieldTypes); |
||
305 | } |
||
306 | |||
307 | 47638 | jjdelcerro | public boolean isAllFieldsDeclareType() { |
308 | return this.all_fields_declare_type; |
||
309 | } |
||
310 | 47636 | fdiaz | } |