svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.csv / src / main / java / org / gvsig / fmap / dal / store / simplereader / SimpleReaderFeatureTypeLoader.java @ 47652
History | View | Annotate | Download (12.5 KB)
1 |
/*
|
---|---|
2 |
* To change this license header, choose License Headers in Project Properties.
|
3 |
* To change this template file, choose Tools | Templates
|
4 |
* and open the template in the editor.
|
5 |
*/
|
6 |
package org.gvsig.fmap.dal.store.simplereader; |
7 |
|
8 |
import java.io.IOException; |
9 |
import java.io.InputStreamReader; |
10 |
import java.io.Reader; |
11 |
import java.util.Locale; |
12 |
import java.util.Map; |
13 |
import org.apache.commons.io.IOUtils; |
14 |
import org.apache.commons.lang3.ArrayUtils; |
15 |
import org.apache.commons.lang3.StringUtils; |
16 |
import org.gvsig.fmap.dal.DataTypes; |
17 |
import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor; |
18 |
import org.gvsig.fmap.dal.feature.EditableFeatureType; |
19 |
import org.gvsig.fmap.dal.store.simplereader.simplereaders.SimpleReader; |
20 |
import org.gvsig.tools.dynobject.Tags; |
21 |
import org.gvsig.tools.task.SimpleTaskStatus; |
22 |
import org.slf4j.Logger; |
23 |
import org.slf4j.LoggerFactory; |
24 |
|
25 |
/**
|
26 |
*
|
27 |
* @author fdiaz
|
28 |
*/
|
29 |
public abstract class SimpleReaderFeatureTypeLoader { |
30 |
|
31 |
private static final Logger LOGGER = LoggerFactory.getLogger(SimpleReaderFeatureTypeLoader.class); |
32 |
|
33 |
protected boolean all_fields_declare_type; |
34 |
protected final SimpleReaderStoreParameters parameters; |
35 |
|
36 |
public SimpleReaderFeatureTypeLoader(SimpleReaderStoreParameters parameters) {
|
37 |
this.parameters = parameters;
|
38 |
} |
39 |
|
40 |
protected abstract String getProviderName(); |
41 |
protected abstract SimpleReader getSimpleReader(Reader in) throws IOException; |
42 |
|
43 |
protected FieldTypeParser[] getFieldTypes(String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) { |
44 |
String fullFileName = parameters.getFile()==null? "":parameters.getFile().getAbsolutePath(); |
45 |
FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length]; |
46 |
|
47 |
for (int i = 0; i < fieldTypes.length; i++) { |
48 |
fieldTypes[i] = new FieldTypeParser(getProviderName(), fullFileName);
|
49 |
} |
50 |
|
51 |
if (automaticTypes != null) { |
52 |
for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) { |
53 |
fieldTypes[i].detectedValue = automaticTypes[i]; |
54 |
fieldTypes[i].type = automaticTypes[i].getType(); |
55 |
} |
56 |
} |
57 |
|
58 |
this.all_fields_declare_type = true; |
59 |
for (int i = 0; i < fieldTypes.length; i++) { |
60 |
if (!fieldTypes[i].parse(headers[i])) {
|
61 |
LOGGER.warn("Can't parse header of field "+i+ "( "+headers[i]+") in '"+getProviderName()+"' file '" + fullFileName + "'."); |
62 |
} |
63 |
if( fieldTypes[i].type == DataTypes.UNKNOWN ) {
|
64 |
all_fields_declare_type = false;
|
65 |
fieldTypes[i].type = DataTypes.STRING; |
66 |
} |
67 |
} |
68 |
|
69 |
String param_types_def = SimpleReaderStoreParameters.getRawFieldTypes(parameters);
|
70 |
if (StringUtils.isNotBlank(param_types_def)) {
|
71 |
String sep = SimpleReaderStoreParameters.getDelimiter(param_types_def);
|
72 |
if (StringUtils.isNotBlank(sep)) {
|
73 |
String[] param_types = param_types_def.split(sep); |
74 |
FieldTypeParser parser = new FieldTypeParser(getProviderName(), fullFileName);
|
75 |
for (String param_type : param_types) { |
76 |
parser.clear(); |
77 |
parser.parse(param_type); |
78 |
for (FieldTypeParser fieldType : fieldTypes) {
|
79 |
if (StringUtils.equalsIgnoreCase(fieldType.name, parser.name)) {
|
80 |
fieldType.copyFrom(parser); |
81 |
break;
|
82 |
} |
83 |
} |
84 |
} |
85 |
} |
86 |
} |
87 |
|
88 |
return fieldTypes;
|
89 |
} |
90 |
|
91 |
protected void fillFeatureType(EditableFeatureType fType, FieldTypeParser[] fieldTypes) { |
92 |
String fullFileName = parameters.getFile()==null? "":parameters.getFile().getAbsolutePath(); |
93 |
|
94 |
Tags ftypeTags = fType.getTags(); |
95 |
for (FieldTypeParser fieldType : fieldTypes) {
|
96 |
EditableFeatureAttributeDescriptor fad = fType.add(fieldType.name, fieldType.type); |
97 |
if( fieldType.detectedValue!=null ) { |
98 |
fad.setDisplaySize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));
|
99 |
fad.setSize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));
|
100 |
if( fad.getPrecision()<fieldType.detectedValue.getPrecision() ) {
|
101 |
fad.setPrecision(fieldType.detectedValue.getPrecision()); |
102 |
} |
103 |
if( fad.getScale()<fieldType.detectedValue.getScale()) {
|
104 |
fad.setScale(fieldType.detectedValue.getScale()); |
105 |
} |
106 |
} else {
|
107 |
fad.setDisplaySize(fieldType.size); |
108 |
} |
109 |
if (fieldType.type == DataTypes.GEOMETRY ) {
|
110 |
fad.setGeometryType(fieldType.geomType, fieldType.geomSubtype); |
111 |
if( fType.getDefaultGeometryAttributeName() == null ) { |
112 |
fType.setDefaultGeometryAttributeName(fieldType.name); |
113 |
} |
114 |
} |
115 |
Locale locale = null; |
116 |
if (fieldType.type == DataTypes.TIMESTAMP ) {
|
117 |
if(!SimpleReaderStoreParameters.isBlankOrDefaultLocale(parameters)){
|
118 |
locale = SimpleReaderStoreParameters.getLocale(parameters); |
119 |
} |
120 |
} else {
|
121 |
locale = SimpleReaderStoreParameters.getLocale(parameters); |
122 |
} |
123 |
fad.setLocale(locale); |
124 |
for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) { |
125 |
try {
|
126 |
switch(entry.getKey().toLowerCase()) {
|
127 |
case "expression": |
128 |
// Los campos calculados los procesamos en una segunda
|
129 |
// pasada, cuando ya estan definidos el resto de los campos
|
130 |
// ya que pueden requerir campos que aun no se han definido.
|
131 |
break;
|
132 |
default:
|
133 |
fad.set(entry.getKey(), entry.getValue()); |
134 |
} |
135 |
} catch (Exception ex) { |
136 |
LOGGER.warn("Can't set property '"+entry.getKey()+"' of '"+fad.getName()+"'.", ex); |
137 |
} |
138 |
} |
139 |
Tags tags = fad.getTags(); |
140 |
for (Map.Entry<String, String> entry : fieldType.tags.entrySet()) { |
141 |
tags.set(entry.getKey(), entry.getValue()); |
142 |
} |
143 |
for (Map.Entry<String, String> entry : fieldType.typetags.entrySet()) { |
144 |
ftypeTags.set(entry.getKey(), entry.getValue()); |
145 |
} |
146 |
for (Map.Entry<String, String> entry : fieldType.typeAssignments.entrySet()) { |
147 |
try {
|
148 |
fType.set(entry.getKey(), entry.getValue()); |
149 |
} catch(Exception ex) { |
150 |
LOGGER.warn("Can't set attribute '"+entry.getKey()+"' in the feature type.", ex); |
151 |
} |
152 |
} |
153 |
} |
154 |
|
155 |
// Processamos ahora los campos calculados
|
156 |
for (FieldTypeParser fieldType : fieldTypes) {
|
157 |
EditableFeatureAttributeDescriptor fad = fType.getEditableAttributeDescriptor(fieldType.name); |
158 |
for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) { |
159 |
try {
|
160 |
switch(entry.getKey().toLowerCase()) {
|
161 |
case "expression": |
162 |
fad.set(entry.getKey(), entry.getValue()); |
163 |
break;
|
164 |
} |
165 |
} catch (Exception ex) { |
166 |
LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+fullFileName+"'.", ex); |
167 |
} |
168 |
} |
169 |
} |
170 |
} |
171 |
|
172 |
public boolean loadFeatureType(EditableFeatureType featureType, SimpleTaskStatus status) throws IOException { |
173 |
return loadFeatureType(featureType, true, status); |
174 |
} |
175 |
|
176 |
public boolean loadFeatureType(EditableFeatureType featureType, boolean detectTypes, SimpleTaskStatus status) throws IOException { |
177 |
InputStreamReader in = null; |
178 |
SimpleReader reader = null;
|
179 |
try {
|
180 |
in = SimpleReaderUtils.openFile( |
181 |
parameters.getFile(), |
182 |
SimpleReaderStoreParameters.getCharset(parameters) |
183 |
); |
184 |
|
185 |
reader = getSimpleReader(in); |
186 |
|
187 |
String[] headers = getHeaders(reader); |
188 |
|
189 |
AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = null; |
190 |
if( detectTypes ) {
|
191 |
detectedTypes = automaticDetectionOfTypes(headers, status); |
192 |
} |
193 |
if( StringUtils.isBlank(headers[headers.length-1]) && |
194 |
(detectedTypes==null || detectedTypes[headers.length-1].isBlank()) ) { |
195 |
headers = ArrayUtils.remove(headers, headers.length-1);
|
196 |
} |
197 |
if (detectedTypes != null && detectedTypes.length > headers.length) { |
198 |
// Se han detectado mas columnas que las que hay en la cabezera,
|
199 |
// a?adimos mas columnas a la cabezera.
|
200 |
String[] headers2 = new String[detectedTypes.length]; |
201 |
for (int i = 0; i < headers2.length; i++) { |
202 |
if (i < headers.length) {
|
203 |
headers2[i] = headers[i]; |
204 |
} else {
|
205 |
headers2[i] = getFixedHeader(i); |
206 |
} |
207 |
} |
208 |
headers = headers2; |
209 |
} |
210 |
for (int i = 0; i < headers.length; i++) { |
211 |
if (StringUtils.isBlank(headers[i])) {
|
212 |
headers[i] = getFixedHeader(i); |
213 |
} |
214 |
} |
215 |
// Initialize the feature types
|
216 |
fillFeatureType(featureType, headers, detectedTypes); |
217 |
return this.all_fields_declare_type; |
218 |
} finally {
|
219 |
IOUtils.closeQuietly(in); |
220 |
IOUtils.closeQuietly(reader); |
221 |
} |
222 |
} |
223 |
|
224 |
protected String[] getHeaders(SimpleReader reader) throws RuntimeException, IOException { |
225 |
String headers[]; |
226 |
headers = SimpleReaderStoreParameters.getHeaders(parameters); |
227 |
return headers;
|
228 |
} |
229 |
|
230 |
protected String[] getFixedHeaders(int count) { |
231 |
String[] headers = new String[count]; |
232 |
for (int i = 0; i < headers.length; i++) { |
233 |
headers[i] = getFixedHeader(i); |
234 |
} |
235 |
return headers;
|
236 |
} |
237 |
|
238 |
|
239 |
protected String getFixedHeader(int column) { |
240 |
char[] header = new char[3]; |
241 |
|
242 |
String s = String.format("%03d", column); |
243 |
header[0] = (char) (s.charAt(0) + 17); |
244 |
header[1] = (char) (s.charAt(1) + 17); |
245 |
header[2] = (char) (s.charAt(2) + 17); |
246 |
return String.valueOf(header); |
247 |
} |
248 |
|
249 |
private AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(String[] headers, SimpleTaskStatus status) throws IOException { |
250 |
String fullFileName = parameters.getFile()==null? "NULL":parameters.getFile().getAbsolutePath(); |
251 |
boolean automatic_types_detection = SimpleReaderStoreParameters.getAutomaticTypesDetection(parameters);
|
252 |
if (!automatic_types_detection) {
|
253 |
return null; |
254 |
} |
255 |
AutomaticDetectionOfTypes.DetectedValue[] types = null; |
256 |
|
257 |
Reader in = null; |
258 |
SimpleReader reader = null;
|
259 |
|
260 |
try {
|
261 |
in = SimpleReaderUtils.openFile( |
262 |
parameters.getFile(), |
263 |
SimpleReaderStoreParameters.getCharset(parameters) |
264 |
); |
265 |
reader = getSimpleReader(in); |
266 |
AutomaticDetectionOfTypes x = new AutomaticDetectionOfTypes(
|
267 |
fullFileName |
268 |
); |
269 |
types = x.detect( |
270 |
headers.length, |
271 |
reader, |
272 |
isFirstLineHeader(), |
273 |
SimpleReaderStoreParameters.getLocale(parameters), |
274 |
status |
275 |
); |
276 |
} catch (Exception ex) { |
277 |
int lineno = 0; |
278 |
if (reader != null) { |
279 |
lineno = reader.getLine(); |
280 |
} |
281 |
throw new RuntimeException("Problems reading file '" + fullFileName + "' near line " + lineno + ".", ex); |
282 |
|
283 |
} finally {
|
284 |
IOUtils.closeQuietly(reader); |
285 |
IOUtils.closeQuietly(in); |
286 |
} |
287 |
return types;
|
288 |
} |
289 |
|
290 |
protected boolean isFirstLineHeader() { |
291 |
return false; |
292 |
} |
293 |
|
294 |
protected void fillFeatureType(EditableFeatureType fType, String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) { |
295 |
fType.setHasOID(true);
|
296 |
FieldTypeParser[] fieldTypes = getFieldTypes(headers, automaticTypes);
|
297 |
fillFeatureType(fType, fieldTypes); |
298 |
} |
299 |
|
300 |
public boolean isAllFieldsDeclareType() { |
301 |
return this.all_fields_declare_type; |
302 |
} |
303 |
} |