svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.csv / src / main / java / org / gvsig / fmap / dal / store / csv / CSVStoreProvider.java @ 45567
History | View | Annotate | Download (48.7 KB)
1 | 40846 | jjdelcerro | /**
|
---|---|---|---|
2 | * gvSIG. Desktop Geographic Information System.
|
||
3 | *
|
||
4 | * Copyright (C) 2007-2013 gvSIG Association.
|
||
5 | *
|
||
6 | 42775 | jjdelcerro | * This program is free software; you can redistribute it and/or modify it under
|
7 | * the terms of the GNU General Public License as published by the Free Software
|
||
8 | * Foundation; either version 3 of the License, or (at your option) any later
|
||
9 | * version.
|
||
10 | 40846 | jjdelcerro | *
|
11 | 42775 | jjdelcerro | * This program is distributed in the hope that it will be useful, but WITHOUT
|
12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||
13 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||
14 | * details.
|
||
15 | 40846 | jjdelcerro | *
|
16 | 42775 | jjdelcerro | * You should have received a copy of the GNU General Public License along with
|
17 | * this program; if not, write to the Free Software Foundation, Inc., 51
|
||
18 | * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||
19 | 40846 | jjdelcerro | *
|
20 | 42775 | jjdelcerro | * For any additional information, do not hesitate to contact us at info AT
|
21 | * gvsig.com, or visit our website www.gvsig.com.
|
||
22 | 40846 | jjdelcerro | */
|
23 | package org.gvsig.fmap.dal.store.csv; |
||
24 | |||
25 | import java.io.File; |
||
26 | 43283 | jjdelcerro | import java.io.FileInputStream; |
27 | import java.io.FileNotFoundException; |
||
28 | 40846 | jjdelcerro | import java.io.FileReader; |
29 | import java.io.IOException; |
||
30 | 43283 | jjdelcerro | import java.io.InputStreamReader; |
31 | 43425 | jjdelcerro | import java.net.URI; |
32 | 41062 | jjdelcerro | import java.net.URL; |
33 | 43283 | jjdelcerro | import java.nio.charset.Charset; |
34 | 40846 | jjdelcerro | import java.util.ArrayList; |
35 | import java.util.HashMap; |
||
36 | 41006 | jjdelcerro | import java.util.Iterator; |
37 | 40846 | jjdelcerro | import java.util.List; |
38 | 41069 | jjdelcerro | import java.util.Locale; |
39 | 44408 | jjdelcerro | import java.util.Map; |
40 | 45425 | jjdelcerro | import javax.json.JsonArray; |
41 | import javax.json.JsonObject; |
||
42 | import javax.json.JsonValue; |
||
43 | 40846 | jjdelcerro | |
44 | 41006 | jjdelcerro | import org.apache.commons.io.FilenameUtils; |
45 | 42775 | jjdelcerro | import org.apache.commons.io.IOUtils; |
46 | 41876 | jjdelcerro | import org.apache.commons.lang3.StringUtils; |
47 | 40846 | jjdelcerro | import org.cresques.cts.IProjection; |
48 | import org.gvsig.fmap.dal.DALLocator; |
||
49 | import org.gvsig.fmap.dal.DataManager; |
||
50 | import org.gvsig.fmap.dal.DataServerExplorer; |
||
51 | import org.gvsig.fmap.dal.DataStore; |
||
52 | import org.gvsig.fmap.dal.DataStoreNotification; |
||
53 | import org.gvsig.fmap.dal.DataTypes; |
||
54 | import org.gvsig.fmap.dal.FileHelper; |
||
55 | import org.gvsig.fmap.dal.exception.DataException; |
||
56 | import org.gvsig.fmap.dal.exception.InitializeException; |
||
57 | import org.gvsig.fmap.dal.exception.OpenException; |
||
58 | import org.gvsig.fmap.dal.exception.ReadException; |
||
59 | import org.gvsig.fmap.dal.exception.ValidateDataParametersException; |
||
60 | 41335 | jjdelcerro | import org.gvsig.fmap.dal.feature.EditableFeature; |
61 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor; |
62 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.feature.EditableFeatureType; |
63 | 41059 | jjdelcerro | import org.gvsig.fmap.dal.feature.Feature; |
64 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor; |
65 | 41335 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureAttributeEmulator; |
66 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureSet; |
67 | 41059 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureStore; |
68 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureType; |
69 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.feature.exception.PerformEditingException; |
70 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.feature.spi.FeatureProvider; |
71 | import org.gvsig.fmap.dal.feature.spi.FeatureStoreProviderServices; |
||
72 | import org.gvsig.fmap.dal.feature.spi.memory.AbstractMemoryStoreProvider; |
||
73 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.resource.ResourceAction; |
74 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.resource.file.FileResource; |
75 | import org.gvsig.fmap.dal.resource.spi.ResourceConsumer; |
||
76 | import org.gvsig.fmap.dal.resource.spi.ResourceProvider; |
||
77 | import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorer; |
||
78 | import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorerParameters; |
||
79 | import org.gvsig.fmap.dal.spi.DataStoreProviderServices; |
||
80 | 44936 | jjdelcerro | import org.gvsig.fmap.dal.store.csv.AutomaticDetectionOfTypes.DetectedValue; |
81 | 41617 | jjdelcerro | import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReader; |
82 | import org.gvsig.fmap.dal.store.csv.simplereaders.FixedLenReader; |
||
83 | 45563 | jolivas | import org.gvsig.fmap.dal.store.csv.simplereaders.JSonReader; |
84 | 41617 | jjdelcerro | import org.gvsig.fmap.dal.store.csv.simplereaders.SimpleReader; |
85 | 40846 | jjdelcerro | import org.gvsig.fmap.geom.Geometry; |
86 | import org.gvsig.fmap.geom.GeometryLocator; |
||
87 | import org.gvsig.fmap.geom.GeometryManager; |
||
88 | 44408 | jjdelcerro | import org.gvsig.fmap.geom.GeometryUtils; |
89 | 41335 | jjdelcerro | import org.gvsig.fmap.geom.aggregate.MultiPoint; |
90 | 40846 | jjdelcerro | import org.gvsig.fmap.geom.primitive.Envelope; |
91 | 41006 | jjdelcerro | import org.gvsig.fmap.geom.primitive.Point; |
92 | import org.gvsig.fmap.geom.type.GeometryType; |
||
93 | 45425 | jjdelcerro | import org.gvsig.json.Json; |
94 | import org.gvsig.json.JsonObjectBuilder; |
||
95 | 40846 | jjdelcerro | import org.gvsig.tools.ToolsLocator; |
96 | 41335 | jjdelcerro | import org.gvsig.tools.dataTypes.DataType; |
97 | 41006 | jjdelcerro | import org.gvsig.tools.dataTypes.DataTypesManager; |
98 | 44669 | jjdelcerro | import org.gvsig.tools.dataTypes.Coercion; |
99 | 41006 | jjdelcerro | import org.gvsig.tools.dispose.DisposableIterator; |
100 | 44408 | jjdelcerro | import org.gvsig.tools.dynobject.Tags; |
101 | 40846 | jjdelcerro | import org.gvsig.tools.dynobject.exception.DynFieldNotFoundException; |
102 | 41006 | jjdelcerro | import org.gvsig.tools.evaluator.AbstractEvaluator; |
103 | import org.gvsig.tools.evaluator.EvaluatorData; |
||
104 | import org.gvsig.tools.evaluator.EvaluatorException; |
||
105 | 41059 | jjdelcerro | import org.gvsig.tools.exception.BaseException; |
106 | 40846 | jjdelcerro | import org.gvsig.tools.exception.NotYetImplemented; |
107 | import org.gvsig.tools.persistence.PersistentState; |
||
108 | import org.gvsig.tools.persistence.exception.PersistenceException; |
||
109 | import org.gvsig.tools.task.SimpleTaskStatus; |
||
110 | import org.gvsig.tools.task.TaskStatusManager; |
||
111 | 41059 | jjdelcerro | import org.gvsig.tools.visitor.VisitCanceledException; |
112 | import org.gvsig.tools.visitor.Visitor; |
||
113 | 40846 | jjdelcerro | import org.slf4j.Logger; |
114 | import org.slf4j.LoggerFactory; |
||
115 | 40878 | jjdelcerro | import org.supercsv.prefs.CsvPreference; |
116 | 44669 | jjdelcerro | import org.gvsig.tools.dataTypes.CoercionContext; |
117 | 40846 | jjdelcerro | |
118 | 44309 | omartinez | @SuppressWarnings("UseSpecificCatch") |
119 | 40846 | jjdelcerro | public class CSVStoreProvider extends AbstractMemoryStoreProvider implements |
120 | 41617 | jjdelcerro | ResourceConsumer { |
121 | |||
122 | 44408 | jjdelcerro | private static final Logger LOGGER = LoggerFactory.getLogger(CSVStoreProvider.class); |
123 | 40846 | jjdelcerro | |
124 | 44669 | jjdelcerro | public static final String NAME = DataStore.CSV_PROVIDER_NAME; |
125 | 40846 | jjdelcerro | public static final String DESCRIPTION = "CSV file"; |
126 | |||
127 | public static final String METADATA_DEFINITION_NAME = NAME; |
||
128 | |||
129 | private ResourceProvider resource;
|
||
130 | |||
131 | private long counterNewsOIDs = 0; |
||
132 | private Envelope envelope;
|
||
133 | 41059 | jjdelcerro | private boolean need_calculate_envelope = false; |
134 | 44309 | omartinez | private final SimpleTaskStatus taskStatus; |
135 | private final CSVFeatureWriter writer; |
||
136 | 45359 | omartinez | private FeatureType featureType;
|
137 | 40846 | jjdelcerro | |
138 | public CSVStoreProvider(CSVStoreParameters parameters,
|
||
139 | 41617 | jjdelcerro | DataStoreProviderServices storeServices) throws InitializeException {
|
140 | 40846 | jjdelcerro | super(
|
141 | 41617 | jjdelcerro | parameters, |
142 | storeServices, |
||
143 | FileHelper.newMetadataContainer(METADATA_DEFINITION_NAME) |
||
144 | 40846 | jjdelcerro | ); |
145 | 44309 | omartinez | this.writer = new CSVFeatureWriter(); |
146 | 40846 | jjdelcerro | TaskStatusManager manager = ToolsLocator.getTaskStatusManager(); |
147 | this.taskStatus = manager.createDefaultSimpleTaskStatus("CSV"); |
||
148 | |||
149 | counterNewsOIDs = 0;
|
||
150 | |||
151 | File file = getCSVParameters().getFile();
|
||
152 | resource = this.createResource(
|
||
153 | 41617 | jjdelcerro | FileResource.NAME, |
154 | new Object[]{file.getAbsolutePath()} |
||
155 | 40846 | jjdelcerro | ); |
156 | |||
157 | resource.addConsumer(this);
|
||
158 | 44309 | omartinez | |
159 | 41006 | jjdelcerro | initializeFeatureTypes(); |
160 | 40846 | jjdelcerro | } |
161 | |||
162 | private CSVStoreParameters getCSVParameters() {
|
||
163 | return (CSVStoreParameters) this.getParameters(); |
||
164 | } |
||
165 | |||
166 | 44309 | omartinez | @Override
|
167 | 40846 | jjdelcerro | public String getProviderName() { |
168 | return NAME;
|
||
169 | } |
||
170 | |||
171 | 44309 | omartinez | @Override
|
172 | 40846 | jjdelcerro | public boolean allowWrite() { |
173 | 44309 | omartinez | return true; |
174 | 40846 | jjdelcerro | } |
175 | |||
176 | 41006 | jjdelcerro | private String getFullFileName() { |
177 | 41617 | jjdelcerro | // Usar solo para mostrar mensajes en el logger.
|
178 | 44309 | omartinez | String s;
|
179 | 41617 | jjdelcerro | try {
|
180 | s = getCSVParameters().getFile().getAbsolutePath(); |
||
181 | } catch (Exception e2) { |
||
182 | s = "(unknow)";
|
||
183 | } |
||
184 | return s;
|
||
185 | 41006 | jjdelcerro | } |
186 | 41617 | jjdelcerro | |
187 | 44309 | omartinez | @Override
|
188 | 40846 | jjdelcerro | public void open() throws OpenException { |
189 | 42775 | jjdelcerro | if (this.data != null) { |
190 | 40846 | jjdelcerro | return;
|
191 | } |
||
192 | 43215 | jjdelcerro | this.data = new ArrayList<>(); |
193 | 41006 | jjdelcerro | resource.setData(new HashMap()); |
194 | counterNewsOIDs = 0;
|
||
195 | 41617 | jjdelcerro | try {
|
196 | loadFeatures(); |
||
197 | } catch (RuntimeException e) { |
||
198 | 44408 | jjdelcerro | LOGGER.debug("Can't load features from CSV '" + getFullFileName() + "'.", e); |
199 | 41617 | jjdelcerro | throw e;
|
200 | } catch (Exception e) { |
||
201 | 44408 | jjdelcerro | LOGGER.debug("Can't load features from CSV '" + getFullFileName() + "'.", e); |
202 | 41617 | jjdelcerro | throw new RuntimeException(e); |
203 | } |
||
204 | 40846 | jjdelcerro | } |
205 | |||
206 | 44309 | omartinez | @Override
|
207 | 40846 | jjdelcerro | public DataServerExplorer getExplorer() throws ReadException { |
208 | DataManager manager = DALLocator.getDataManager(); |
||
209 | 41006 | jjdelcerro | FilesystemServerExplorerParameters params; |
210 | 40846 | jjdelcerro | try {
|
211 | params = (FilesystemServerExplorerParameters) manager |
||
212 | 41617 | jjdelcerro | .createServerExplorerParameters(FilesystemServerExplorer.NAME); |
213 | 40846 | jjdelcerro | params.setRoot(this.getCSVParameters().getFile().getParent());
|
214 | 41617 | jjdelcerro | return manager.openServerExplorer(FilesystemServerExplorer.NAME, params);
|
215 | 40846 | jjdelcerro | } catch (DataException e) {
|
216 | throw new ReadException(this.getProviderName(), e); |
||
217 | } catch (ValidateDataParametersException e) {
|
||
218 | throw new ReadException(this.getProviderName(), e); |
||
219 | } |
||
220 | |||
221 | } |
||
222 | |||
223 | 44309 | omartinez | @Override
|
224 | 40846 | jjdelcerro | public void performChanges(Iterator deleteds, Iterator inserteds, Iterator updateds, Iterator originalFeatureTypesUpdated) throws PerformEditingException { |
225 | |||
226 | try {
|
||
227 | this.taskStatus.add();
|
||
228 | taskStatus.message("_preparing");
|
||
229 | 44309 | omartinez | this.getResource().closeRequest();
|
230 | 40846 | jjdelcerro | getResource().execute(new ResourceAction() {
|
231 | 44309 | omartinez | @Override
|
232 | 40846 | jjdelcerro | public Object run() throws Exception { |
233 | FeatureSet features = null;
|
||
234 | DisposableIterator it = null;
|
||
235 | try {
|
||
236 | File file = (File) resource.get(); |
||
237 | 41617 | jjdelcerro | |
238 | features |
||
239 | = getStoreServices().getFeatureStore() |
||
240 | 44309 | omartinez | .getFeatureSet(); |
241 | 41006 | jjdelcerro | List<FeatureProvider> newdata = new ArrayList<FeatureProvider>(); |
242 | 44309 | omartinez | FeatureType ftype = getStoreServices().getDefaultFeatureType(); |
243 | writer.initialize(getCSVParameters(), file, ftype, getCSVPreferences()); |
||
244 | 44900 | omartinez | writer.begin(); |
245 | 40846 | jjdelcerro | it = features.fastIterator(); |
246 | 41617 | jjdelcerro | taskStatus.setRangeOfValues(0, 0); |
247 | long counter = 0; |
||
248 | 42775 | jjdelcerro | while (it.hasNext()) {
|
249 | 40846 | jjdelcerro | taskStatus.setCurValue(counter++); |
250 | FeatureProvider feature = getStoreServices().getFeatureProviderFromFeature( |
||
251 | 41617 | jjdelcerro | (org.gvsig.fmap.dal.feature.Feature) it.next()); |
252 | 40846 | jjdelcerro | writer.add(feature); |
253 | 42775 | jjdelcerro | if (feature.getOID() == null) { |
254 | 44408 | jjdelcerro | LOGGER.warn("feature without OID");
|
255 | 40846 | jjdelcerro | feature.setOID(createNewOID()); |
256 | } |
||
257 | newdata.add(feature); |
||
258 | } |
||
259 | data = newdata; |
||
260 | 42775 | jjdelcerro | if (writer.getEnvelope() != null) { |
261 | 40846 | jjdelcerro | envelope = writer.getEnvelope().getGeometry().getEnvelope(); |
262 | } |
||
263 | resource.notifyChanges(); |
||
264 | 41006 | jjdelcerro | writer.end(); |
265 | 40846 | jjdelcerro | } finally {
|
266 | 42775 | jjdelcerro | if (it != null) { |
267 | 40846 | jjdelcerro | it.dispose(); |
268 | } |
||
269 | 42775 | jjdelcerro | if (features != null) { |
270 | 40846 | jjdelcerro | features.dispose(); |
271 | } |
||
272 | } |
||
273 | return null; |
||
274 | } |
||
275 | 41617 | jjdelcerro | |
276 | 40846 | jjdelcerro | }); |
277 | this.taskStatus.terminate();
|
||
278 | } catch (Exception e) { |
||
279 | this.taskStatus.abort();
|
||
280 | throw new PerformEditingException(getResource().toString(), e); |
||
281 | } finally {
|
||
282 | this.taskStatus.remove();
|
||
283 | } |
||
284 | } |
||
285 | |||
286 | 44309 | omartinez | private CsvPreference getCSVPreferences() {
|
287 | CSVReader reader = new CSVReader(getCSVParameters());
|
||
288 | return reader.getCSVPreferences();
|
||
289 | } |
||
290 | |||
291 | @Override
|
||
292 | 40846 | jjdelcerro | public boolean closeResourceRequested(ResourceProvider resource) { |
293 | return true; |
||
294 | } |
||
295 | |||
296 | 44309 | omartinez | @Override
|
297 | 40846 | jjdelcerro | public int getOIDType() { |
298 | return DataTypes.LONG;
|
||
299 | } |
||
300 | |||
301 | 44309 | omartinez | @Override
|
302 | 40846 | jjdelcerro | public boolean supportsAppendMode() { |
303 | 44309 | omartinez | return true; |
304 | 40846 | jjdelcerro | } |
305 | |||
306 | 44309 | omartinez | @Override
|
307 | public void append(final FeatureProvider featureProvider) { |
||
308 | //todo
|
||
309 | getResource().execute(new ResourceAction() {
|
||
310 | @Override
|
||
311 | public Object run() throws Exception { |
||
312 | //writer.append(getStoreServices().createFeature(featureProvider));
|
||
313 | writer.add(featureProvider); |
||
314 | return null; |
||
315 | } |
||
316 | }); |
||
317 | 40846 | jjdelcerro | } |
318 | |||
319 | 44309 | omartinez | @Override
|
320 | public void beginAppend() throws DataException { |
||
321 | this.close();
|
||
322 | getResource().execute(new ResourceAction() {
|
||
323 | public Object run() throws Exception { |
||
324 | writer.initialize( |
||
325 | getCSVParameters(), |
||
326 | getCSVParameters().getFile(), |
||
327 | getFeatureStore().getDefaultFeatureType(), |
||
328 | 44388 | omartinez | getCSVPreferences() |
329 | 44309 | omartinez | ); |
330 | writer.beginAppend(); |
||
331 | return null; |
||
332 | } |
||
333 | }); |
||
334 | |||
335 | 40846 | jjdelcerro | } |
336 | |||
337 | 44309 | omartinez | @Override
|
338 | 40846 | jjdelcerro | public void endAppend() { |
339 | 44309 | omartinez | try {
|
340 | getResource().execute(new ResourceAction() {
|
||
341 | public Object run() throws Exception { |
||
342 | writer.end(); |
||
343 | resource.notifyChanges(); //resourcesNotifyChanges();
|
||
344 | counterNewsOIDs = -1;
|
||
345 | return null; |
||
346 | } |
||
347 | }); |
||
348 | writer.end(); |
||
349 | } catch (PerformEditingException ex) {
|
||
350 | 44408 | jjdelcerro | LOGGER.warn("Not been able to end append '" + this.getFullName() + "'.", ex); |
351 | 44309 | omartinez | } |
352 | 40846 | jjdelcerro | } |
353 | |||
354 | public void saveToState(PersistentState state) throws PersistenceException { |
||
355 | throw new NotYetImplemented(); |
||
356 | } |
||
357 | |||
358 | public void loadFromState(PersistentState state) throws PersistenceException { |
||
359 | throw new NotYetImplemented(); |
||
360 | } |
||
361 | |||
362 | 44309 | omartinez | @Override
|
363 | 40846 | jjdelcerro | public Object createNewOID() { |
364 | 44309 | omartinez | return counterNewsOIDs++;
|
365 | 40846 | jjdelcerro | } |
366 | |||
367 | protected void initializeFeatureTypes() throws InitializeException { |
||
368 | try {
|
||
369 | this.open();
|
||
370 | } catch (OpenException e) {
|
||
371 | throw new InitializeException(this.getProviderName(), e); |
||
372 | } |
||
373 | } |
||
374 | |||
375 | 44309 | omartinez | @Override
|
376 | 40846 | jjdelcerro | public Envelope getEnvelope() throws DataException { |
377 | this.open();
|
||
378 | 42775 | jjdelcerro | if (this.envelope != null) { |
379 | 41617 | jjdelcerro | return this.envelope; |
380 | 41059 | jjdelcerro | } |
381 | 42775 | jjdelcerro | if (!this.need_calculate_envelope) { |
382 | 41617 | jjdelcerro | return null; |
383 | 41059 | jjdelcerro | } |
384 | FeatureStore fs = this.getFeatureStore();
|
||
385 | FeatureType ft = fs.getDefaultFeatureType(); |
||
386 | FeatureAttributeDescriptor fad = ft.getAttributeDescriptor(ft.getDefaultGeometryAttributeIndex()); |
||
387 | |||
388 | try {
|
||
389 | this.envelope = GeometryLocator.getGeometryManager().createEnvelope(fad.getGeomType().getSubType());
|
||
390 | 41617 | jjdelcerro | fs.accept(new Visitor() {
|
391 | 44309 | omartinez | @Override
|
392 | 41617 | jjdelcerro | public void visit(Object obj) throws VisitCanceledException, BaseException { |
393 | Feature f = (Feature) obj; |
||
394 | Geometry geom = f.getDefaultGeometry(); |
||
395 | 42775 | jjdelcerro | if (geom != null) { |
396 | 41617 | jjdelcerro | envelope.add(geom.getEnvelope()); |
397 | } |
||
398 | } |
||
399 | }); |
||
400 | } catch (BaseException e) {
|
||
401 | 44408 | jjdelcerro | LOGGER.warn("Can't calculate the envelope of CSV file '" + this.getFullName() + "'.", e); |
402 | 41617 | jjdelcerro | this.envelope = null; |
403 | } |
||
404 | |||
405 | 41059 | jjdelcerro | this.need_calculate_envelope = false; |
406 | 40846 | jjdelcerro | return this.envelope; |
407 | } |
||
408 | |||
409 | 44309 | omartinez | @Override
|
410 | 40846 | jjdelcerro | public Object getDynValue(String name) throws DynFieldNotFoundException { |
411 | 42775 | jjdelcerro | if (DataStore.METADATA_ENVELOPE.equalsIgnoreCase(name)) {
|
412 | 40846 | jjdelcerro | try {
|
413 | return this.getEnvelope(); |
||
414 | } catch (DataException e) {
|
||
415 | return null; |
||
416 | } |
||
417 | } else {
|
||
418 | 42775 | jjdelcerro | if (DataStore.METADATA_CRS.equalsIgnoreCase(name)) {
|
419 | 40878 | jjdelcerro | IProjection pro = CSVStoreParameters.getCRS(this.getCSVParameters());
|
420 | 42775 | jjdelcerro | if (pro != null) { |
421 | 40846 | jjdelcerro | return pro;
|
422 | } |
||
423 | } |
||
424 | } |
||
425 | return super.getDynValue(name); |
||
426 | } |
||
427 | |||
428 | 44309 | omartinez | @Override
|
429 | 40846 | jjdelcerro | public void resourceChanged(ResourceProvider resource) { |
430 | this.getStoreServices().notifyChange(
|
||
431 | 41617 | jjdelcerro | DataStoreNotification.RESOURCE_CHANGED, |
432 | resource); |
||
433 | 40846 | jjdelcerro | } |
434 | |||
435 | 44309 | omartinez | @Override
|
436 | 40846 | jjdelcerro | public Object getSourceId() { |
437 | return this.getCSVParameters().getFile(); |
||
438 | } |
||
439 | |||
440 | 44309 | omartinez | @Override
|
441 | 40846 | jjdelcerro | public String getName() { |
442 | 40878 | jjdelcerro | String name = this.getCSVParameters().getFile().getName(); |
443 | 41006 | jjdelcerro | return FilenameUtils.getBaseName(name);
|
444 | 40846 | jjdelcerro | } |
445 | |||
446 | 44309 | omartinez | @Override
|
447 | 40846 | jjdelcerro | public String getFullName() { |
448 | return this.getCSVParameters().getFile().getAbsolutePath(); |
||
449 | } |
||
450 | |||
451 | 44309 | omartinez | @Override
|
452 | 40846 | jjdelcerro | public ResourceProvider getResource() {
|
453 | return resource;
|
||
454 | } |
||
455 | |||
456 | private boolean isEmpty(String s) { |
||
457 | 42775 | jjdelcerro | if (s == null) { |
458 | 41617 | jjdelcerro | return true; |
459 | } |
||
460 | return s.trim().length() == 0; |
||
461 | 40846 | jjdelcerro | } |
462 | 40878 | jjdelcerro | |
463 | 44309 | omartinez | private void init(CSVStoreParameters parameters, DataStoreProviderServices storeServices) { |
464 | throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. |
||
465 | } |
||
466 | 44941 | jjdelcerro | |
467 | public static class FieldTypeParser { |
||
468 | 41617 | jjdelcerro | |
469 | 41335 | jjdelcerro | public String name = null; |
470 | public int type = DataTypes.STRING; |
||
471 | public int size = 0; |
||
472 | 44408 | jjdelcerro | public int geomType = Geometry.TYPES.GEOMETRY; |
473 | public int geomSubtype = Geometry.SUBTYPES.GEOM2D; |
||
474 | public Map<String,String> tags = new HashMap<>(); |
||
475 | public Map<String,String> assignments = new HashMap<>(); |
||
476 | 44936 | jjdelcerro | |
477 | // Valores obtenidos desde la deteccion automatica desde los datos
|
||
478 | public DetectedValue detectedValue = null ; |
||
479 | 41335 | jjdelcerro | |
480 | private String typename = "string"; |
||
481 | 41617 | jjdelcerro | |
482 | 44941 | jjdelcerro | private final String fullFileName; |
483 | private final String providerName; |
||
484 | |||
485 | FieldTypeParser(String providerName, String fullFileName) { |
||
486 | this.providerName = providerName;
|
||
487 | this.fullFileName = fullFileName;
|
||
488 | 41335 | jjdelcerro | } |
489 | 41617 | jjdelcerro | |
490 | 44941 | jjdelcerro | public String getProviderName() { |
491 | return this.providerName; |
||
492 | } |
||
493 | |||
494 | public String getFullFileName() { |
||
495 | return this.fullFileName; |
||
496 | } |
||
497 | |||
498 | 44408 | jjdelcerro | public void clear() { |
499 | this.name = null; |
||
500 | this.type = DataTypes.STRING;
|
||
501 | this.size = 0; |
||
502 | this.tags = new HashMap<>(); |
||
503 | this.assignments = new HashMap<>(); |
||
504 | } |
||
505 | |||
506 | public void copyFrom(FieldTypeParser other) { |
||
507 | this.name = other.name;
|
||
508 | this.type = other.type;
|
||
509 | this.size = other.size;
|
||
510 | this.tags = new HashMap<>(); |
||
511 | this.tags.putAll(other.tags);
|
||
512 | this.assignments = new HashMap<>(); |
||
513 | this.assignments.putAll(other.assignments);
|
||
514 | } |
||
515 | |||
516 | 41335 | jjdelcerro | private int getType(String value) { |
517 | 41617 | jjdelcerro | DataTypesManager dataTypesManager = ToolsLocator.getDataTypesManager(); |
518 | 41335 | jjdelcerro | return dataTypesManager.getType(typename);
|
519 | } |
||
520 | 41617 | jjdelcerro | |
521 | 44408 | jjdelcerro | private String[] split(String value, String separators) { |
522 | int firstSeparatorPosition = 1000000; |
||
523 | Character sep = null; |
||
524 | for (char ch : separators.toCharArray()) { |
||
525 | int pos = value.indexOf(ch);
|
||
526 | if( pos>0 && pos<firstSeparatorPosition ) { |
||
527 | sep = ch; |
||
528 | firstSeparatorPosition = pos; |
||
529 | } |
||
530 | } |
||
531 | if( sep == null ) { |
||
532 | return new String[] { value }; |
||
533 | } |
||
534 | return value.split("["+sep+"]"); |
||
535 | 42775 | jjdelcerro | } |
536 | |||
537 | 44408 | jjdelcerro | @SuppressWarnings("UseSpecificCatch") |
538 | 41335 | jjdelcerro | public boolean parse(String value) { |
539 | 44408 | jjdelcerro | String[] args; |
540 | if ( value.contains("__") ) { |
||
541 | args = value.split("__");
|
||
542 | } else {
|
||
543 | args = split(value, ":/#@!;-");
|
||
544 | if( args.length == 1 ) { |
||
545 | this.name = value;
|
||
546 | return true; |
||
547 | } |
||
548 | } |
||
549 | int n = 0; |
||
550 | 44445 | jjdelcerro | this.name = StringUtils.trim(args[n++]);
|
551 | 44408 | jjdelcerro | if( n >= args.length ) {
|
552 | 41335 | jjdelcerro | return true; |
553 | } |
||
554 | 44445 | jjdelcerro | this.typename = StringUtils.trim(args[n++]);
|
555 | 44408 | jjdelcerro | this.type = this.getType(this.typename); |
556 | if ( this.type == DataTypes.INVALID ) { |
||
557 | this.geomType = GeometryUtils.getGeometryType(this.typename); |
||
558 | if( this.geomType==Geometry.TYPES.UNKNOWN ) { |
||
559 | this.type = DataTypes.STRING;
|
||
560 | LOGGER.info("Type '" + this.typename + "' not valid for attribute '" + value + "' in '"+getProviderName()+"' file '" + getFullFileName() + "'."); |
||
561 | } else {
|
||
562 | this.typename = "GEOMETRY"; |
||
563 | this.type = DataTypes.GEOMETRY;
|
||
564 | } |
||
565 | } |
||
566 | 44669 | jjdelcerro | this.size = 0; |
567 | 44498 | omartinez | |
568 | 44408 | jjdelcerro | while (n < args.length) {
|
569 | 44445 | jjdelcerro | String option = StringUtils.trim(args[n++].toLowerCase());
|
570 | 44408 | jjdelcerro | switch (option) {
|
571 | case "size": |
||
572 | try {
|
||
573 | this.size = Integer.parseInt(args[n++]); |
||
574 | } catch (Exception ex) { |
||
575 | LOGGER.warn("Ignore incorrect field size for field " + value + " in '"+getProviderName()+"' file '" + getFullFileName() + "'.", ex); |
||
576 | 42775 | jjdelcerro | } |
577 | 44408 | jjdelcerro | break;
|
578 | case "tag": { |
||
579 | 44445 | jjdelcerro | String x = StringUtils.trim(args[n++]);
|
580 | 44408 | jjdelcerro | int pos = x.indexOf("="); |
581 | if( pos < 0 ) { |
||
582 | this.tags.put(x, null); |
||
583 | } else {
|
||
584 | this.tags.put(
|
||
585 | 44445 | jjdelcerro | StringUtils.trim(StringUtils.substring(x, 0, pos)),
|
586 | StringUtils.trim(StringUtils.substring(x, pos+1))
|
||
587 | 44408 | jjdelcerro | ); |
588 | } |
||
589 | break;
|
||
590 | 42775 | jjdelcerro | } |
591 | 44408 | jjdelcerro | case "set": { |
592 | 44445 | jjdelcerro | String x = StringUtils.trim(args[n++]);
|
593 | 44408 | jjdelcerro | int pos = x.indexOf("="); |
594 | if( pos < 0 ) { |
||
595 | this.assignments.put(x, null); |
||
596 | } else {
|
||
597 | this.assignments.put(
|
||
598 | 44445 | jjdelcerro | StringUtils.trim(StringUtils.substring(x, 0, pos)),
|
599 | StringUtils.trim(StringUtils.substring(x, pos+1))
|
||
600 | 44408 | jjdelcerro | ); |
601 | 42775 | jjdelcerro | } |
602 | 44408 | jjdelcerro | break;
|
603 | 41876 | jjdelcerro | } |
604 | 44408 | jjdelcerro | default:
|
605 | 44738 | jjdelcerro | if( StringUtils.isNumeric(option) && this.size == 0 ) { |
606 | try {
|
||
607 | this.size = Integer.parseInt(option); |
||
608 | } catch (Exception ex) { |
||
609 | LOGGER.warn("Ignore incorrect field size for field " + value + " in '"+getProviderName()+"' file '" + getFullFileName() + "'.", ex); |
||
610 | } |
||
611 | break;
|
||
612 | } |
||
613 | 44408 | jjdelcerro | LOGGER.warn("Illegal argumente '"+option+"' for field '"+this.name+"' in '"+getProviderName()+"' file '" + getFullFileName() + "' ("+value+")."); |
614 | } |
||
615 | 41335 | jjdelcerro | } |
616 | return true; |
||
617 | } |
||
618 | 41617 | jjdelcerro | |
619 | 41335 | jjdelcerro | } |
620 | 41617 | jjdelcerro | |
621 | 44936 | jjdelcerro | private EditableFeatureType getFeatureType(String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) { |
622 | 41335 | jjdelcerro | EditableFeatureType fType = getStoreServices().createFeatureType(this.getName());
|
623 | fType.setHasOID(true);
|
||
624 | |||
625 | FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length]; |
||
626 | 41617 | jjdelcerro | //
|
627 | 41335 | jjdelcerro | // Calculamos cuales pueden ser los tipos de datos
|
628 | //
|
||
629 | 42775 | jjdelcerro | for (int i = 0; i < fieldTypes.length; i++) { |
630 | 44941 | jjdelcerro | fieldTypes[i] = new FieldTypeParser(getProviderName(), getFullFileName());
|
631 | 41335 | jjdelcerro | } |
632 | |||
633 | 41617 | jjdelcerro | // Asuminos los tipos pasados por parametro, que se supone
|
634 | 41335 | jjdelcerro | // son los detectados automaticamente.
|
635 | 42775 | jjdelcerro | if (automaticTypes != null) { |
636 | for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) { |
||
637 | 44936 | jjdelcerro | fieldTypes[i].detectedValue = automaticTypes[i]; |
638 | fieldTypes[i].type = automaticTypes[i].getType(); |
||
639 | 41335 | jjdelcerro | } |
640 | } |
||
641 | // Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo
|
||
642 | // los tipos anteriores en caso de definirse en la cabezara.
|
||
643 | 42775 | jjdelcerro | for (int i = 0; i < fieldTypes.length; i++) { |
644 | if (!fieldTypes[i].parse(headers[i])) {
|
||
645 | 41335 | jjdelcerro | continue;
|
646 | } |
||
647 | |||
648 | } |
||
649 | |||
650 | 41617 | jjdelcerro | // Y por ultimo hacemos caso a lo que se haya especificado en los parametros
|
651 | 41335 | jjdelcerro | // de apertura del CSV, teniendo esto prioridad sobre todo.
|
652 | 41876 | jjdelcerro | String param_types_def = CSVStoreParameters.getRawFieldTypes(this.getParameters()); |
653 | 41901 | jjdelcerro | if (StringUtils.isNotBlank(param_types_def)) {
|
654 | String sep = CSVStoreParameters.getDelimiter(param_types_def);
|
||
655 | 42775 | jjdelcerro | if (StringUtils.isNotBlank(sep)) {
|
656 | 41901 | jjdelcerro | String[] param_types = param_types_def.split(sep); |
657 | 44941 | jjdelcerro | FieldTypeParser parser = new FieldTypeParser(getProviderName(), getFullFileName());
|
658 | 42775 | jjdelcerro | for (String param_type : param_types) { |
659 | parser.clear(); |
||
660 | parser.parse(param_type); |
||
661 | for (FieldTypeParser fieldType : fieldTypes) {
|
||
662 | if (StringUtils.equalsIgnoreCase(fieldType.name, parser.name)) {
|
||
663 | fieldType.copyFrom(parser); |
||
664 | break;
|
||
665 | } |
||
666 | 41901 | jjdelcerro | } |
667 | 41335 | jjdelcerro | } |
668 | } |
||
669 | } |
||
670 | 41617 | jjdelcerro | //
|
671 | 41335 | jjdelcerro | // Una vez ya sabemos los tipos de datos rellenamos el feature-type
|
672 | //
|
||
673 | 44408 | jjdelcerro | for (FieldTypeParser fieldType : fieldTypes) {
|
674 | EditableFeatureAttributeDescriptor fad = fType.add(fieldType.name, fieldType.type); |
||
675 | 44941 | jjdelcerro | if( fieldType.detectedValue!=null ) { |
676 | 44947 | jjdelcerro | fad.setDisplaySize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));
|
677 | 44941 | jjdelcerro | fad.setSize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));
|
678 | if( fad.getPrecision()<fieldType.detectedValue.getPrecision() ) {
|
||
679 | fad.setPrecision(fieldType.detectedValue.getPrecision()); |
||
680 | } |
||
681 | if( fad.getScale()<fieldType.detectedValue.getScale()) {
|
||
682 | fad.setScale(fieldType.detectedValue.getScale()); |
||
683 | } |
||
684 | } else {
|
||
685 | 44947 | jjdelcerro | fad.setDisplaySize(fieldType.size); |
686 | 44936 | jjdelcerro | } |
687 | 44408 | jjdelcerro | if (fieldType.type == DataTypes.GEOMETRY ) {
|
688 | fad.setGeometryType(fieldType.geomType, fieldType.geomSubtype); |
||
689 | if( fType.getDefaultGeometryAttributeName() == null ) { |
||
690 | fType.setDefaultGeometryAttributeName(fieldType.name); |
||
691 | 41876 | jjdelcerro | } |
692 | 41335 | jjdelcerro | } |
693 | 44894 | omartinez | Locale locale = CSVStoreParameters.getLocale(this.getParameters()); |
694 | fad.setLocale(locale); |
||
695 | 44408 | jjdelcerro | for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) { |
696 | try {
|
||
697 | switch(entry.getKey().toLowerCase()) {
|
||
698 | case "expression": |
||
699 | // Los campos calculados los procesamos en una segunda
|
||
700 | // pasada, cuando ya estan definidos el resto de los campos
|
||
701 | // ya que pueden requerir campos que aun no se han definido.
|
||
702 | break;
|
||
703 | default:
|
||
704 | 45359 | omartinez | fad.set(entry.getKey(), entry.getValue()); |
705 | } |
||
706 | 44408 | jjdelcerro | } catch (Exception ex) { |
707 | LOGGER.warn("Can't set property '"+entry.getKey()+"' of '"+fad.getName()+"'.", ex); |
||
708 | } |
||
709 | 44894 | omartinez | } |
710 | 44408 | jjdelcerro | Tags tags = fad.getTags(); |
711 | for (Map.Entry<String, String> entry : fieldType.tags.entrySet()) { |
||
712 | tags.set(entry.getKey(), entry.getValue()); |
||
713 | } |
||
714 | 41335 | jjdelcerro | } |
715 | 44408 | jjdelcerro | // Processamos ahora los campos calculados
|
716 | for (FieldTypeParser fieldType : fieldTypes) {
|
||
717 | EditableFeatureAttributeDescriptor fad = fType.getEditableAttributeDescriptor(fieldType.name); |
||
718 | for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) { |
||
719 | try {
|
||
720 | switch(entry.getKey().toLowerCase()) {
|
||
721 | case "expression": |
||
722 | fad.set(entry.getKey(), entry.getValue()); |
||
723 | break;
|
||
724 | } |
||
725 | } catch (Exception ex) { |
||
726 | LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+getFullFileName()+"'.", ex); |
||
727 | } |
||
728 | } |
||
729 | } |
||
730 | 41335 | jjdelcerro | String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters()); |
731 | 44408 | jjdelcerro | if ( pointDimensionNames != null ) { |
732 | 41335 | jjdelcerro | PointAttributeEmulator emulator = new PointAttributeEmulator(pointDimensionNames);
|
733 | 44936 | jjdelcerro | String columnName = CSVStoreParameters.getPointColumnName(this.getParameters()); |
734 | if( StringUtils.isBlank(columnName) ) {
|
||
735 | columnName = "the_geom";
|
||
736 | } |
||
737 | EditableFeatureAttributeDescriptor attr = fType.add(columnName, DataTypes.GEOMETRY, emulator); |
||
738 | 41335 | jjdelcerro | GeometryManager geommgr = GeometryLocator.getGeometryManager(); |
739 | GeometryType gt; |
||
740 | try {
|
||
741 | 44408 | jjdelcerro | if ( emulator.fieldNames != null && emulator.fieldNames.length <= 2 ) { |
742 | gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM2D); |
||
743 | } else {
|
||
744 | gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM3D); |
||
745 | } |
||
746 | 41335 | jjdelcerro | attr.setGeometryType(gt); |
747 | } catch (Exception e) { |
||
748 | 44408 | jjdelcerro | LOGGER.warn("Can't set geometry type for the calculated field in '"+getProviderName()+"' file '" + getFullFileName() + "'.", e); |
749 | 41335 | jjdelcerro | } |
750 | 44408 | jjdelcerro | } |
751 | |||
752 | 43283 | jjdelcerro | String geometry_column = CSVStoreParameters.getGeometryColumn(this.getParameters()); |
753 | 44309 | omartinez | if (!StringUtils.isEmpty(geometry_column)) {
|
754 | 43283 | jjdelcerro | EditableFeatureAttributeDescriptor attr = (EditableFeatureAttributeDescriptor) fType.get(geometry_column); |
755 | 44309 | omartinez | if (attr != null && attr.getType() != DataTypes.GEOMETRY) { |
756 | 43283 | jjdelcerro | attr.setDataType(DataTypes.GEOMETRY); |
757 | GeometryManager geommgr = GeometryLocator.getGeometryManager(); |
||
758 | GeometryType gt; |
||
759 | try {
|
||
760 | gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.UNKNOWN); |
||
761 | attr.setGeometryType(gt); |
||
762 | } catch (Exception e) { |
||
763 | 44408 | jjdelcerro | LOGGER.warn("Can't set geometry type for the calculated field in CSV file '" + getFullFileName() + "'.", e); |
764 | 43283 | jjdelcerro | } |
765 | fType.setDefaultGeometryAttributeName(geometry_column); |
||
766 | } |
||
767 | } |
||
768 | 41335 | jjdelcerro | return fType;
|
769 | 40846 | jjdelcerro | } |
770 | 41617 | jjdelcerro | |
771 | 41335 | jjdelcerro | static class PointAttributeEmulator implements FeatureAttributeEmulator { |
772 | |||
773 | private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class); |
||
774 | |||
775 | 41342 | jjdelcerro | private static final int XNAME = 0; |
776 | private static final int YNAME = 1; |
||
777 | private static final int ZNAME = 2; |
||
778 | 41617 | jjdelcerro | |
779 | 45425 | jjdelcerro | private GeometryManager geommgr;
|
780 | private String[] fieldNames; |
||
781 | private Coercion toDouble;
|
||
782 | private DataType dataType;
|
||
783 | 41335 | jjdelcerro | private int errorcount = 0; |
784 | |||
785 | 45425 | jjdelcerro | public PointAttributeEmulator() {
|
786 | |||
787 | } |
||
788 | |||
789 | 41335 | jjdelcerro | public PointAttributeEmulator(String[] pointDimensionNames) { |
790 | 42775 | jjdelcerro | if (pointDimensionNames.length > 2) { |
791 | 41342 | jjdelcerro | this.fieldNames = new String[3]; |
792 | this.fieldNames[ZNAME] = pointDimensionNames[2]; |
||
793 | 41335 | jjdelcerro | } else {
|
794 | 41342 | jjdelcerro | this.fieldNames = new String[2]; |
795 | 41335 | jjdelcerro | } |
796 | 41342 | jjdelcerro | this.fieldNames[XNAME] = pointDimensionNames[0]; |
797 | this.fieldNames[YNAME] = pointDimensionNames[1]; |
||
798 | 41335 | jjdelcerro | this.geommgr = GeometryLocator.getGeometryManager();
|
799 | DataTypesManager datatypeManager = ToolsLocator.getDataTypesManager(); |
||
800 | |||
801 | this.toDouble = datatypeManager.getCoercion(DataTypes.DOUBLE);
|
||
802 | this.dataType = datatypeManager.get(DataTypes.GEOMETRY);
|
||
803 | } |
||
804 | 41617 | jjdelcerro | |
805 | 44309 | omartinez | @Override
|
806 | 41335 | jjdelcerro | public Object get(Feature feature) { |
807 | try {
|
||
808 | 41348 | jjdelcerro | Object valueX = feature.get(this.fieldNames[XNAME]); |
809 | valueX = toDouble.coerce(valueX); |
||
810 | 42775 | jjdelcerro | if (valueX == null) { |
811 | 41348 | jjdelcerro | return null; |
812 | } |
||
813 | Object valueY = feature.get(this.fieldNames[YNAME]); |
||
814 | valueY = toDouble.coerce(valueY); |
||
815 | 42775 | jjdelcerro | if (valueY == null) { |
816 | 41348 | jjdelcerro | return null; |
817 | } |
||
818 | Object valueZ = null; |
||
819 | 42775 | jjdelcerro | if (this.fieldNames.length > 2) { |
820 | 41348 | jjdelcerro | valueZ = toDouble.coerce(feature.get(this.fieldNames[ZNAME]));
|
821 | 42775 | jjdelcerro | if (valueZ == null) { |
822 | 41348 | jjdelcerro | return null; |
823 | } |
||
824 | } |
||
825 | 41617 | jjdelcerro | |
826 | 44309 | omartinez | double x = ((Double) valueX); |
827 | double y = ((Double) valueY); |
||
828 | 41335 | jjdelcerro | Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);
|
829 | 42775 | jjdelcerro | if (this.fieldNames.length > 2) { |
830 | 44309 | omartinez | double z = ((Double) valueZ); |
831 | 41335 | jjdelcerro | point.setCoordinateAt(2, z);
|
832 | } |
||
833 | return point;
|
||
834 | } catch (Exception ex) { |
||
835 | 42775 | jjdelcerro | if (++errorcount < 5) { |
836 | 41335 | jjdelcerro | logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='" |
837 | 41617 | jjdelcerro | + this.fieldNames[XNAME] + "', YNAME='" + this.fieldNames[XNAME] + "' feature=" + feature.toString(), ex); |
838 | 41335 | jjdelcerro | } |
839 | return null; |
||
840 | } |
||
841 | } |
||
842 | |||
843 | public void set(EditableFeature feature, Object value) { |
||
844 | 42775 | jjdelcerro | if (value == null) { |
845 | 41335 | jjdelcerro | return;
|
846 | } |
||
847 | Point point = null; |
||
848 | 42775 | jjdelcerro | if (value instanceof MultiPoint) { |
849 | 41617 | jjdelcerro | point = (Point) ((MultiPoint) value).getPrimitiveAt(0); |
850 | 41335 | jjdelcerro | } else {
|
851 | point = (Point) value;
|
||
852 | } |
||
853 | 41342 | jjdelcerro | feature.set(this.fieldNames[XNAME], point.getX());
|
854 | feature.set(this.fieldNames[YNAME], point.getY());
|
||
855 | 42775 | jjdelcerro | if (this.fieldNames.length > 2) { |
856 | 41342 | jjdelcerro | feature.set(this.fieldNames[ZNAME], point.getCoordinateAt(2)); |
857 | 41335 | jjdelcerro | } |
858 | } |
||
859 | |||
860 | 44309 | omartinez | @Override
|
861 | 41335 | jjdelcerro | public boolean allowSetting() { |
862 | return true; |
||
863 | } |
||
864 | 41342 | jjdelcerro | |
865 | 44309 | omartinez | @Override
|
866 | 41342 | jjdelcerro | public String[] getRequiredFieldNames() { |
867 | return this.fieldNames; |
||
868 | } |
||
869 | 41617 | jjdelcerro | |
870 | 45425 | jjdelcerro | @Override
|
871 | public JsonObject toJson() {
|
||
872 | return this.toJsonBuilder().build(); |
||
873 | } |
||
874 | |||
875 | @Override
|
||
876 | public JsonObjectBuilder toJsonBuilder() {
|
||
877 | JsonObjectBuilder builder = Json.createObjectBuilder(); |
||
878 | builder.add_class(this);
|
||
879 | builder.add("fieldNames", fieldNames);
|
||
880 | return builder;
|
||
881 | } |
||
882 | |||
883 | @Override
|
||
884 | public void fromJson(JsonObject json) { |
||
885 | JsonArray jsonFieldNames = json.getJsonArray("fieldNames");
|
||
886 | this.fieldNames = new String[jsonFieldNames.size()]; |
||
887 | for (int i = 0; i < jsonFieldNames.size(); i++) { |
||
888 | this.fieldNames[XNAME] = jsonFieldNames.getString(i);
|
||
889 | } |
||
890 | this.geommgr = GeometryLocator.getGeometryManager();
|
||
891 | DataTypesManager datatypeManager = ToolsLocator.getDataTypesManager(); |
||
892 | |||
893 | this.toDouble = datatypeManager.getCoercion(DataTypes.DOUBLE);
|
||
894 | this.dataType = datatypeManager.get(DataTypes.GEOMETRY);
|
||
895 | } |
||
896 | |||
897 | 41335 | jjdelcerro | } |
898 | 41617 | jjdelcerro | |
899 | 41006 | jjdelcerro | static class ToPointEvaluaror extends AbstractEvaluator { |
900 | 40846 | jjdelcerro | |
901 | 41335 | jjdelcerro | private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class); |
902 | 40846 | jjdelcerro | |
903 | 41335 | jjdelcerro | private GeometryManager geommgr = null; |
904 | private String xname = null; |
||
905 | private String yname = null; |
||
906 | private String zname = null; |
||
907 | private Coercion toDouble;
|
||
908 | private int errorcount = 0; |
||
909 | |||
910 | ToPointEvaluaror(String[] pointDimensionNames) { |
||
911 | this.xname = pointDimensionNames[0]; |
||
912 | this.yname = pointDimensionNames[1]; |
||
913 | 42775 | jjdelcerro | if (pointDimensionNames.length > 2) { |
914 | 41335 | jjdelcerro | this.zname = pointDimensionNames[2]; |
915 | } |
||
916 | this.geommgr = GeometryLocator.getGeometryManager();
|
||
917 | this.toDouble = ToolsLocator.getDataTypesManager().getCoercion(DataTypes.DOUBLE);
|
||
918 | } |
||
919 | |||
920 | 44309 | omartinez | @Override
|
921 | 41335 | jjdelcerro | public Object evaluate(EvaluatorData data) throws EvaluatorException { |
922 | try {
|
||
923 | double x = ((Double) toDouble.coerce(data.getDataValue(xname))).doubleValue(); |
||
924 | double y = ((Double) toDouble.coerce(data.getDataValue(yname))).doubleValue(); |
||
925 | Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);
|
||
926 | 42775 | jjdelcerro | if (zname != null) { |
927 | 41335 | jjdelcerro | double z = ((Double) toDouble.coerce(data.getDataValue(zname))).doubleValue(); |
928 | point.setCoordinateAt(2, z);
|
||
929 | } |
||
930 | return point;
|
||
931 | } catch (Exception ex) { |
||
932 | 42775 | jjdelcerro | if (++errorcount < 5) { |
933 | 41335 | jjdelcerro | logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='" |
934 | + xname + "', YNAME='" + yname + "', ZNAME='" + zname + "', data=" + data.toString()); |
||
935 | } |
||
936 | return null; |
||
937 | } |
||
938 | } |
||
939 | |||
940 | 44309 | omartinez | @Override
|
941 | 41335 | jjdelcerro | public String getName() { |
942 | return "ToPointEvaluaror"; |
||
943 | } |
||
944 | |||
945 | 40846 | jjdelcerro | } |
946 | 41335 | jjdelcerro | |
947 | 45563 | jolivas | private SimpleReader getSimpleReader(InputStreamReader in) throws IOException { |
948 | 41617 | jjdelcerro | SimpleReader reader; |
949 | 45563 | jolivas | String filename = CSVStoreParameters.getFileName(getCSVParameters());
|
950 | if (FilenameUtils.isExtension(filename, "json")){ |
||
951 | 45567 | jolivas | // reader= new JSonReader(in);
|
952 | reader= new JSonReader(in,getCSVParameters());
|
||
953 | 45563 | jolivas | } else if (CSVStoreParameters.getRawFieldsDefinition(getCSVParameters()) != null) { |
954 | 41617 | jjdelcerro | reader = new FixedLenReader(in, getCSVParameters());
|
955 | } else {
|
||
956 | reader = new CSVReader(in, getCSVParameters());
|
||
957 | } |
||
958 | return reader;
|
||
959 | } |
||
960 | 41006 | jjdelcerro | |
961 | 42775 | jjdelcerro | private String getFixedHeader(int column) { |
962 | char[] header = new char[3]; |
||
963 | |||
964 | String s = String.format("%03d", column); |
||
965 | header[0] = (char) (s.charAt(0) + 17); |
||
966 | header[1] = (char) (s.charAt(1) + 17); |
||
967 | header[2] = (char) (s.charAt(2) + 17); |
||
968 | return String.valueOf(header); |
||
969 | } |
||
970 | |||
971 | private String[] getFixedHeaders(int count) { |
||
972 | String[] headers = new String[count]; |
||
973 | for (int i = 0; i < headers.length; i++) { |
||
974 | headers[i] = getFixedHeader(i); |
||
975 | } |
||
976 | return headers;
|
||
977 | } |
||
978 | |||
979 | 43283 | jjdelcerro | private InputStreamReader openFile(File f, String charsetName) throws FileNotFoundException { |
980 | Charset charset = Charset.defaultCharset(); |
||
981 | FileInputStream fis = new FileInputStream(f); |
||
982 | 44309 | omartinez | if (!StringUtils.isEmpty(charsetName)) {
|
983 | if (Charset.isSupported(charsetName)) { |
||
984 | 43283 | jjdelcerro | try {
|
985 | charset = Charset.forName(charsetName);
|
||
986 | 44309 | omartinez | } catch (Throwable th) { |
987 | 44408 | jjdelcerro | LOGGER.warn("Can't use charset '" + charsetName + "' for read csv '" + this.getFullFileName() + "'.", th); |
988 | 43283 | jjdelcerro | } |
989 | } else {
|
||
990 | 44408 | jjdelcerro | LOGGER.warn("charset '" + charsetName + "' not supported for read csv '" + this.getFullFileName() + "'."); |
991 | 43283 | jjdelcerro | } |
992 | } |
||
993 | InputStreamReader isr = new InputStreamReader(fis, charset); |
||
994 | return isr;
|
||
995 | } |
||
996 | 44309 | omartinez | |
997 | 42775 | jjdelcerro | private void loadFeatures() { |
998 | 43283 | jjdelcerro | InputStreamReader in = null; |
999 | 41617 | jjdelcerro | SimpleReader reader = null;
|
1000 | try {
|
||
1001 | String headers[] = null; |
||
1002 | FeatureStoreProviderServices store = this.getStoreServices();
|
||
1003 | 41006 | jjdelcerro | |
1004 | 41617 | jjdelcerro | boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());
|
1005 | 42220 | fdiaz | |
1006 | 43283 | jjdelcerro | in = openFile( |
1007 | 44309 | omartinez | this.getCSVParameters().getFile(),
|
1008 | CSVStoreParameters.getCharset(this.getCSVParameters())
|
||
1009 | 43283 | jjdelcerro | ); |
1010 | 41006 | jjdelcerro | |
1011 | 41617 | jjdelcerro | reader = getSimpleReader(in); |
1012 | 41006 | jjdelcerro | |
1013 | 41617 | jjdelcerro | headers = CSVStoreParameters.getHeaders(getCSVParameters()); |
1014 | 42775 | jjdelcerro | if (headers == null) { |
1015 | if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
|
||
1016 | headers = reader.getHeader(); |
||
1017 | if (headers == null) { |
||
1018 | if (CSVStoreParameters.getIgnoreErrors(getCSVParameters())) {
|
||
1019 | headers = getFixedHeaders(reader.getColumnsCount()); |
||
1020 | } else {
|
||
1021 | String msg = "Can't retrieve header from csv file '" |
||
1022 | + this.getCSVParameters().getFile()
|
||
1023 | 44309 | omartinez | .getAbsolutePath() |
1024 | 42775 | jjdelcerro | + "' and not specified in the parameters.";
|
1025 | 44408 | jjdelcerro | LOGGER.warn(msg); |
1026 | 42775 | jjdelcerro | throw new RuntimeException(msg); |
1027 | } |
||
1028 | } |
||
1029 | } else {
|
||
1030 | headers = getFixedHeaders(reader.getColumnsCount()); |
||
1031 | 41617 | jjdelcerro | } |
1032 | 42775 | jjdelcerro | } else {
|
1033 | if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
|
||
1034 | reader.getHeader(); // Skip and ignore the header of file
|
||
1035 | } |
||
1036 | 41617 | jjdelcerro | } |
1037 | 41006 | jjdelcerro | |
1038 | 44936 | jjdelcerro | AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = automaticDetectionOfTypes(headers);
|
1039 | 44309 | omartinez | if (detectedTypes != null && detectedTypes.length > headers.length) { |
1040 | 42775 | jjdelcerro | // Se han detectado mas columnas que las que hay en la cabezera,
|
1041 | // a?adimos mas columnas a la cabezera.
|
||
1042 | String[] headers2 = new String[detectedTypes.length]; |
||
1043 | 44309 | omartinez | for (int i = 0; i < headers2.length; i++) { |
1044 | if (i < headers.length) {
|
||
1045 | 42775 | jjdelcerro | headers2[i] = headers[i]; |
1046 | } else {
|
||
1047 | headers2[i] = getFixedHeader(i); |
||
1048 | } |
||
1049 | } |
||
1050 | headers = headers2; |
||
1051 | } |
||
1052 | 44309 | omartinez | for (int i = 0; i < headers.length; i++) { |
1053 | if (StringUtils.isEmpty(headers[i])) {
|
||
1054 | 44272 | jjdelcerro | headers[i] = getFixedHeader(i); |
1055 | } |
||
1056 | } |
||
1057 | 41617 | jjdelcerro | // Initialize the feature types
|
1058 | 42775 | jjdelcerro | EditableFeatureType edftype = this.getFeatureType(headers, detectedTypes);
|
1059 | 41617 | jjdelcerro | FeatureType ftype = edftype.getNotEditableCopy(); |
1060 | 45359 | omartinez | this.setFeatureType(ftype);
|
1061 | 41059 | jjdelcerro | |
1062 | 41617 | jjdelcerro | Coercion coercion[] = new Coercion[ftype.size()]; |
1063 | 44669 | jjdelcerro | CoercionContext coercionContext[] = new CoercionContext[ftype.size()]; |
1064 | 41617 | jjdelcerro | int sizes[] = new int[ftype.size()]; |
1065 | 42775 | jjdelcerro | for (int i = 0; i < ftype.size(); i++) { |
1066 | 41617 | jjdelcerro | sizes[i] = -1;
|
1067 | FeatureAttributeDescriptor ad = ftype.getAttributeDescriptor(i); |
||
1068 | 44669 | jjdelcerro | coercion[i] = ad.getCoercion(); |
1069 | coercionContext[i] = ad.getCoercionContext(); |
||
1070 | 44309 | omartinez | switch (ad.getDataType().getType()) {
|
1071 | case DataTypes.INT:
|
||
1072 | 44669 | jjdelcerro | sizes[i] = 0;
|
1073 | 44309 | omartinez | break;
|
1074 | case DataTypes.LONG:
|
||
1075 | 44669 | jjdelcerro | sizes[i] = 0;
|
1076 | 44309 | omartinez | break;
|
1077 | case DataTypes.STRING:
|
||
1078 | if (ad.getSize() == 0) { |
||
1079 | // Es un string y no tiene un size asignado.
|
||
1080 | // Lo ponemos a cero para calcularlo.
|
||
1081 | sizes[i] = 0;
|
||
1082 | } |
||
1083 | break;
|
||
1084 | case DataTypes.URL:
|
||
1085 | case DataTypes.URI:
|
||
1086 | case DataTypes.FILE:
|
||
1087 | 41617 | jjdelcerro | sizes[i] = 0;
|
1088 | } |
||
1089 | } |
||
1090 | 42775 | jjdelcerro | if (ftype.getDefaultGeometryAttributeName() != null) { |
1091 | 41617 | jjdelcerro | this.need_calculate_envelope = true; |
1092 | } |
||
1093 | 41006 | jjdelcerro | |
1094 | 41617 | jjdelcerro | Locale locale = CSVStoreParameters.getLocale(getCSVParameters());
|
1095 | taskStatus.message("_loading");
|
||
1096 | int count = 0; |
||
1097 | 41062 | jjdelcerro | |
1098 | 41617 | jjdelcerro | int count_errors = 0; |
1099 | 41062 | jjdelcerro | |
1100 | 41617 | jjdelcerro | List<String> row = reader.read(); |
1101 | 41062 | jjdelcerro | |
1102 | 41617 | jjdelcerro | int skipLines = CSVStoreParameters.getSkipLines(getCSVParameters());
|
1103 | 42775 | jjdelcerro | if (skipLines > 0) { |
1104 | 41617 | jjdelcerro | row = reader.skip(skipLines); |
1105 | } |
||
1106 | 42775 | jjdelcerro | int limit = CSVStoreParameters.getLimit(getCSVParameters());
|
1107 | while (row != null) { |
||
1108 | 41617 | jjdelcerro | taskStatus.setCurValue(++count); |
1109 | FeatureProvider feature = this.createFeatureProvider(ftype);
|
||
1110 | 42775 | jjdelcerro | for (int i = 0; i < row.size(); i++) { |
1111 | 41617 | jjdelcerro | Object rawvalue = row.get(i);
|
1112 | try {
|
||
1113 | Object value = null; |
||
1114 | 44749 | jjdelcerro | if( coercion[i]!=null ) { |
1115 | value = coercion[i].coerce(rawvalue, coercionContext[i]); |
||
1116 | } |
||
1117 | 41617 | jjdelcerro | feature.set(i, value); |
1118 | 44309 | omartinez | if (sizes[i] >= 0 |
1119 | && (value instanceof String || value instanceof URL |
||
1120 | || value instanceof URI || value instanceof File)) { |
||
1121 | 43425 | jjdelcerro | int x = value.toString().length();
|
1122 | 42775 | jjdelcerro | if (sizes[i] < x) {
|
1123 | 41617 | jjdelcerro | sizes[i] = x; |
1124 | } |
||
1125 | } |
||
1126 | 42775 | jjdelcerro | } catch (Exception ex) { |
1127 | if (!ignore_errors) {
|
||
1128 | 41617 | jjdelcerro | throw ex;
|
1129 | } |
||
1130 | 42775 | jjdelcerro | if (count_errors++ < 10) { |
1131 | 44408 | jjdelcerro | LOGGER.warn("Can't load value of attribute " + i + " in row " + count + ".", ex); |
1132 | 41617 | jjdelcerro | } |
1133 | 42775 | jjdelcerro | if (count_errors == 10) { |
1134 | 44408 | jjdelcerro | LOGGER.info("Too many errors, suppress messages.");
|
1135 | 41617 | jjdelcerro | } |
1136 | } |
||
1137 | } |
||
1138 | this.addFeatureProvider(feature);
|
||
1139 | 44309 | omartinez | if (limit > 0) { |
1140 | if (limit < this.data.size()) { |
||
1141 | 42775 | jjdelcerro | break;
|
1142 | } |
||
1143 | } |
||
1144 | 41617 | jjdelcerro | row = reader.read(); |
1145 | } |
||
1146 | 42775 | jjdelcerro | for (int i = 0; i < ftype.size(); i++) { |
1147 | if (sizes[i] > 0) { |
||
1148 | 41617 | jjdelcerro | EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor) edftype.getAttributeDescriptor(i)); |
1149 | efad.setSize(sizes[i]); |
||
1150 | } |
||
1151 | } |
||
1152 | // Volvemos a asignar al store el featuretype, ya que puede
|
||
1153 | // haber cambiado.
|
||
1154 | ftype = edftype.getNotEditableCopy(); |
||
1155 | 45359 | omartinez | this.setFeatureType(ftype);
|
1156 | |||
1157 | 41062 | jjdelcerro | |
1158 | 41617 | jjdelcerro | taskStatus.terminate(); |
1159 | 44963 | jjdelcerro | } catch (Throwable ex) { |
1160 | 41643 | jjdelcerro | int lineno = 0; |
1161 | 42775 | jjdelcerro | if (reader != null) { |
1162 | 41643 | jjdelcerro | lineno = reader.getLine(); |
1163 | } |
||
1164 | 42775 | jjdelcerro | throw new RuntimeException("Problems reading file '" + getFullFileName() + "' near line " + lineno + ".", ex); |
1165 | 41643 | jjdelcerro | |
1166 | 41617 | jjdelcerro | } finally {
|
1167 | 42775 | jjdelcerro | if (reader != null) { |
1168 | 41617 | jjdelcerro | try {
|
1169 | reader.close(); |
||
1170 | } catch (Exception ex) { |
||
1171 | // Do nothing
|
||
1172 | } |
||
1173 | reader = null;
|
||
1174 | } |
||
1175 | 42775 | jjdelcerro | if (in != null) { |
1176 | 41617 | jjdelcerro | try {
|
1177 | in.close(); |
||
1178 | } catch (Exception ex) { |
||
1179 | // Do nothing
|
||
1180 | } |
||
1181 | in = null;
|
||
1182 | } |
||
1183 | } |
||
1184 | } |
||
1185 | 41062 | jjdelcerro | |
1186 | 44936 | jjdelcerro | private AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(String[] headers) throws IOException { |
1187 | 41617 | jjdelcerro | boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(getCSVParameters());
|
1188 | 42775 | jjdelcerro | if (!automatic_types_detection) {
|
1189 | 41617 | jjdelcerro | return null; |
1190 | } |
||
1191 | 44936 | jjdelcerro | AutomaticDetectionOfTypes.DetectedValue[] types = null; |
1192 | 41617 | jjdelcerro | |
1193 | FileReader in = null; |
||
1194 | SimpleReader reader = null;
|
||
1195 | |||
1196 | try {
|
||
1197 | in = new FileReader(this.getCSVParameters().getFile()); |
||
1198 | reader = getSimpleReader(in); |
||
1199 | 43983 | jjdelcerro | AutomaticDetectionOfTypes x = new AutomaticDetectionOfTypes(
|
1200 | this.getFullFileName()
|
||
1201 | ); |
||
1202 | 45567 | jolivas | types = x.detect( |
1203 | headers.length, |
||
1204 | reader, |
||
1205 | CSVStoreParameters.isFirstLineHeader(getCSVParameters()), |
||
1206 | CSVStoreParameters.getLocale(getCSVParameters()) |
||
1207 | ); |
||
1208 | 42775 | jjdelcerro | } catch (Exception ex) { |
1209 | 41643 | jjdelcerro | int lineno = 0; |
1210 | 42775 | jjdelcerro | if (reader != null) { |
1211 | 41643 | jjdelcerro | lineno = reader.getLine(); |
1212 | } |
||
1213 | 42775 | jjdelcerro | throw new RuntimeException("Problems reading file '" + getFullFileName() + "' near line " + lineno + ".", ex); |
1214 | 41643 | jjdelcerro | |
1215 | 41617 | jjdelcerro | } finally {
|
1216 | 42775 | jjdelcerro | IOUtils.closeQuietly(reader); |
1217 | IOUtils.closeQuietly(in); |
||
1218 | } |
||
1219 | return types;
|
||
1220 | } |
||
1221 | 45359 | omartinez | |
1222 | @Override
|
||
1223 | public void fixFeatureTypeFromParameters() { |
||
1224 | String param_types_def = CSVStoreParameters.getRawFieldTypes(this.getParameters()); |
||
1225 | String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters()); |
||
1226 | String geometry_column = CSVStoreParameters.getGeometryColumn(this.getParameters()); |
||
1227 | if (StringUtils.isNotBlank(param_types_def) ||
|
||
1228 | pointDimensionNames != null ||
|
||
1229 | !StringUtils.isBlank(geometry_column)) { |
||
1230 | this.setFeatureType(featureType);
|
||
1231 | } |
||
1232 | } |
||
1233 | |||
1234 | private void setFeatureType(FeatureType ftype) { |
||
1235 | FeatureStoreProviderServices store = this.getStoreServices();
|
||
1236 | List<FeatureType> ftypes = new ArrayList<>(); |
||
1237 | ftypes.add(ftype); |
||
1238 | this.featureType = ftype;
|
||
1239 | store.setFeatureTypes(ftypes, ftype); |
||
1240 | } |
||
1241 | 42775 | jjdelcerro | |
1242 | 40846 | jjdelcerro | } |