Statistics
| Revision:

svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.csv / src / main / java / org / gvsig / fmap / dal / store / csv / CSVStoreProvider.java @ 41643

History | View | Annotate | Download (43.9 KB)

1
/**
2
 * gvSIG. Desktop Geographic Information System.
3
 *
4
 * Copyright (C) 2007-2013 gvSIG Association.
5
 *
6
 * This program is free software; you can redistribute it and/or
7
 * modify it under the terms of the GNU General Public License
8
 * as published by the Free Software Foundation; either version 3
9
 * of the License, or (at your option) any later version.
10
 *
11
 * This program is distributed in the hope that it will be useful,
12
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
 * GNU General Public License for more details.
15
 *
16
 * You should have received a copy of the GNU General Public License
17
 * along with this program; if not, write to the Free Software
18
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
19
 * MA 02110-1301, USA.
20
 *
21
 * For any additional information, do not hesitate to contact us
22
 * at info AT gvsig.com, or visit our website www.gvsig.com.
23
 */
24
package org.gvsig.fmap.dal.store.csv;
25

    
26
import java.io.File;
27
import java.io.FileReader;
28
import java.io.FileWriter;
29
import java.io.IOException;
30
import java.net.URL;
31
import java.text.SimpleDateFormat;
32
import java.util.ArrayList;
33
import java.util.HashMap;
34
import java.util.Iterator;
35
import java.util.List;
36
import java.util.Locale;
37

    
38
import org.apache.commons.io.FilenameUtils;
39
import org.cresques.cts.IProjection;
40
import org.gvsig.fmap.dal.DALLocator;
41
import org.gvsig.fmap.dal.DataManager;
42
import org.gvsig.fmap.dal.DataServerExplorer;
43
import org.gvsig.fmap.dal.DataStore;
44
import org.gvsig.fmap.dal.DataStoreNotification;
45
import org.gvsig.fmap.dal.DataTypes;
46
import org.gvsig.fmap.dal.FileHelper;
47
import org.gvsig.fmap.dal.exception.DataException;
48
import org.gvsig.fmap.dal.exception.InitializeException;
49
import org.gvsig.fmap.dal.exception.OpenException;
50
import org.gvsig.fmap.dal.exception.ReadException;
51
import org.gvsig.fmap.dal.exception.ValidateDataParametersException;
52
import org.gvsig.fmap.dal.feature.EditableFeature;
53
import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;
54
import org.gvsig.fmap.dal.feature.EditableFeatureType;
55
import org.gvsig.fmap.dal.feature.Feature;
56
import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;
57
import org.gvsig.fmap.dal.feature.FeatureAttributeEmulator;
58
import org.gvsig.fmap.dal.feature.FeatureSet;
59
import org.gvsig.fmap.dal.feature.FeatureStore;
60
import org.gvsig.fmap.dal.feature.FeatureType;
61
import org.gvsig.fmap.dal.feature.exception.PerformEditingException;
62
import org.gvsig.fmap.dal.feature.spi.FeatureProvider;
63
import org.gvsig.fmap.dal.feature.spi.FeatureStoreProviderServices;
64
import org.gvsig.fmap.dal.feature.spi.memory.AbstractMemoryStoreProvider;
65
import org.gvsig.fmap.dal.resource.ResourceAction;
66
import org.gvsig.fmap.dal.resource.file.FileResource;
67
import org.gvsig.fmap.dal.resource.spi.ResourceConsumer;
68
import org.gvsig.fmap.dal.resource.spi.ResourceProvider;
69
import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorer;
70
import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorerParameters;
71
import org.gvsig.fmap.dal.spi.DataStoreProviderServices;
72
import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReader;
73
import org.gvsig.fmap.dal.store.csv.simplereaders.FixedLenReader;
74
import org.gvsig.fmap.dal.store.csv.simplereaders.SimpleReader;
75
import org.gvsig.fmap.geom.Geometry;
76
import org.gvsig.fmap.geom.GeometryLocator;
77
import org.gvsig.fmap.geom.GeometryManager;
78
import org.gvsig.fmap.geom.aggregate.MultiPoint;
79
import org.gvsig.fmap.geom.primitive.Envelope;
80
import org.gvsig.fmap.geom.primitive.Point;
81
import org.gvsig.fmap.geom.type.GeometryType;
82
import org.gvsig.tools.ToolsLocator;
83
import org.gvsig.tools.dataTypes.CoercionException;
84
import org.gvsig.tools.dataTypes.DataType;
85
import org.gvsig.tools.dataTypes.DataTypesManager;
86
import org.gvsig.tools.dataTypes.DataTypesManager.Coercion;
87
import org.gvsig.tools.dataTypes.DataTypesManager.CoercionWithLocale;
88
import org.gvsig.tools.dispose.DisposableIterator;
89
import org.gvsig.tools.dynobject.exception.DynFieldNotFoundException;
90
import org.gvsig.tools.evaluator.AbstractEvaluator;
91
import org.gvsig.tools.evaluator.EvaluatorData;
92
import org.gvsig.tools.evaluator.EvaluatorException;
93
import org.gvsig.tools.exception.BaseException;
94
import org.gvsig.tools.exception.NotYetImplemented;
95
import org.gvsig.tools.persistence.PersistentState;
96
import org.gvsig.tools.persistence.exception.PersistenceException;
97
import org.gvsig.tools.task.SimpleTaskStatus;
98
import org.gvsig.tools.task.TaskStatusManager;
99
import org.gvsig.tools.visitor.VisitCanceledException;
100
import org.gvsig.tools.visitor.Visitor;
101
import org.slf4j.Logger;
102
import org.slf4j.LoggerFactory;
103
import org.supercsv.io.CsvListWriter;
104
import org.supercsv.prefs.CsvPreference;
105

    
106
public class CSVStoreProvider extends AbstractMemoryStoreProvider implements
107
        ResourceConsumer {
108

    
109
    private static final Logger logger = LoggerFactory.getLogger(CSVStoreProvider.class);
110

    
111
    public static final String NAME = "CSV";
112
    public static final String DESCRIPTION = "CSV file";
113

    
114
    public static final String METADATA_DEFINITION_NAME = NAME;
115

    
116
    private ResourceProvider resource;
117

    
118
    private long counterNewsOIDs = 0;
119
    private Envelope envelope;
120
    private boolean need_calculate_envelope = false;
121
    private SimpleTaskStatus taskStatus;
122

    
123
    public CSVStoreProvider(CSVStoreParameters parameters,
124
            DataStoreProviderServices storeServices) throws InitializeException {
125
        super(
126
                parameters,
127
                storeServices,
128
                FileHelper.newMetadataContainer(METADATA_DEFINITION_NAME)
129
        );
130

    
131
        TaskStatusManager manager = ToolsLocator.getTaskStatusManager();
132
        this.taskStatus = manager.createDefaultSimpleTaskStatus("CSV");
133

    
134
        counterNewsOIDs = 0;
135

    
136
        File file = getCSVParameters().getFile();
137
        resource = this.createResource(
138
                FileResource.NAME,
139
                new Object[]{file.getAbsolutePath()}
140
        );
141

    
142
        resource.addConsumer(this);
143
        initializeFeatureTypes();
144
    }
145

    
146
    private CSVStoreParameters getCSVParameters() {
147
        return (CSVStoreParameters) this.getParameters();
148
    }
149

    
150
    public String getProviderName() {
151
        return NAME;
152
    }
153

    
154
    public boolean allowWrite() {
155
        return false;
156
    }
157

    
158
    private String getFullFileName() {
159
        // Usar solo para mostrar mensajes en el logger.
160
        String s = "(unknow)";
161
        try {
162
            s = getCSVParameters().getFile().getAbsolutePath();
163
        } catch (Exception e2) {
164
            s = "(unknow)";
165
        }
166
        return s;
167
    }
168

    
169
    public void open() throws OpenException {
170
        if ( this.data != null ) {
171
            return;
172
        }
173
        this.data = new ArrayList<FeatureProvider>();
174
        resource.setData(new HashMap());
175
        counterNewsOIDs = 0;
176
        try {
177
            loadFeatures();
178
        } catch (RuntimeException e) {
179
            logger.warn("Can't load features from CSV '" + getFullFileName() + "'.", e);
180
            throw e;
181
        } catch (Exception e) {
182
            logger.warn("Can't load features from CSV '" + getFullFileName() + "'.", e);
183
            throw new RuntimeException(e);
184
        }
185
    }
186

    
187
    public DataServerExplorer getExplorer() throws ReadException {
188
        DataManager manager = DALLocator.getDataManager();
189
        FilesystemServerExplorerParameters params;
190
        try {
191
            params = (FilesystemServerExplorerParameters) manager
192
                    .createServerExplorerParameters(FilesystemServerExplorer.NAME);
193
            params.setRoot(this.getCSVParameters().getFile().getParent());
194
            return manager.openServerExplorer(FilesystemServerExplorer.NAME, params);
195
        } catch (DataException e) {
196
            throw new ReadException(this.getProviderName(), e);
197
        } catch (ValidateDataParametersException e) {
198
            throw new ReadException(this.getProviderName(), e);
199
        }
200

    
201
    }
202

    
203
    class Writer {
204

    
205
        private Envelope envelope = null;
206
        private boolean calculate_envelope = false;
207
        private CsvListWriter listWriter = null;
208
        private CsvPreference csvpreferences = null;
209
        private FileWriter fwriter = null;
210
        private FeatureType ftype;
211
        private File file;
212
        private String[] values;
213
        private FeatureAttributeDescriptor[] descriptors;
214
        private Coercion convert = null;
215
        private int errorcounts = 0;
216
        private Throwable lasterror = null;
217
        private Locale locale = null;
218

    
219
        public void initialize(File file, FeatureType ftype, CsvPreference csvpreferences) {
220
            this.file = file;
221
            this.ftype = ftype;
222
            this.csvpreferences = csvpreferences;
223
            this.locale = CSVStoreParameters.getLocale(getCSVParameters());
224
            if ( csvpreferences == null ) {
225
                this.csvpreferences = CsvPreference.STANDARD_PREFERENCE;
226
            }
227
            if ( ftype.getDefaultGeometryAttributeName() != null ) {
228
                this.calculate_envelope = true;
229
            }
230
            this.descriptors = this.ftype.getAttributeDescriptors();
231
            this.convert = ToolsLocator.getDataTypesManager().getCoercion(org.gvsig.tools.dataTypes.DataTypes.STRING);
232
            this.errorcounts = 0;
233
        }
234

    
235
        public void begin() {
236
            try {
237
                this.fwriter = new FileWriter(file);
238
            } catch (IOException e) {
239
                logger.warn("Can't open file for write (" + file.getAbsolutePath() + ").", e);
240
                throw new RuntimeException(e);
241
            }
242
            this.listWriter = new CsvListWriter(this.fwriter, this.csvpreferences);
243
            int n = 0;
244
            for ( int i = 0; i < descriptors.length; i++ ) {
245
                FeatureAttributeDescriptor descriptor = descriptors[i];
246
                if ( descriptor.getEvaluator() == null ) {
247
                    n++;
248
                }
249
            }
250

    
251
            String[] header = new String[n];
252
            this.values = new String[n];
253
            n = 0;
254
            for ( int i = 0; i < descriptors.length; i++ ) {
255
                FeatureAttributeDescriptor descriptor = descriptors[i];
256
                if ( descriptor.getEvaluator() == null ) {
257
                    String name = descriptor.getName();
258
                    String typeName = descriptor.getDataTypeName();
259
                    if ( descriptor.getDataType().getType() == DataTypes.STRING ) {
260
                        header[n++] = name + "__" + typeName + "__" + descriptor.getSize();
261
                    } else {
262
                        header[n++] = name + "__" + typeName;
263
                    }
264
                }
265
            }
266
            try {
267
                listWriter.writeHeader(header);
268
            } catch (Exception e) {
269
                logger.warn("Can't write header '" + header.toString() + "' file for write (" + file.getAbsolutePath() + ").", e);
270
                throw new RuntimeException(e);
271
            }
272
        }
273

    
274
        public void add(FeatureProvider feature) {
275
            if ( this.calculate_envelope ) {
276
                Geometry geom = feature.getDefaultGeometry();
277
                if ( geom != null ) {
278
                    if ( envelope == null ) {
279
                        try {
280
                            envelope = (Envelope) geom.getEnvelope().clone();
281
                        } catch (CloneNotSupportedException e) {
282
                            logger.warn("Este error no deberia pasar, siempre se puede hacer un clone de un envelope.", e);
283
                        }
284
                    } else {
285
                        envelope.add(geom.getEnvelope());
286
                    }
287
                }
288
            }
289
            int n = 0;
290
            for ( int i = 0; i < descriptors.length; i++ ) {
291
                FeatureAttributeDescriptor descriptor = descriptors[i];
292
                if ( descriptor.getEvaluator() == null ) {
293
                    Object value = feature.get(i);
294
                    try {
295
                        n++;
296
                        if ( this.convert != null && this.convert instanceof CoercionWithLocale ) {
297
                            values[n] = (String) ((CoercionWithLocale) this.convert).coerce(value, this.locale);
298
                        } else {
299
                            values[n] = (String) this.convert.coerce(value);
300
                        }
301
                    } catch (CoercionException e) {
302
                        try {
303
                            values[n] = value.toString();
304
                        } catch (Exception ex) {
305
                            values[n] = "";
306
                        }
307
                        if ( errorcounts++ <= 10 ) {
308
                            this.lasterror = e;
309
                            logger.warn("Can't convert value of field " + i + " to string in CVS file '" + getFullFileName() + "'.", e);
310
                            if ( errorcounts == 10 ) {
311
                                logger.warn("Too many error writing CVS file '" + getFullFileName() + "', don't output more.");
312
                            }
313
                        }
314
                    }
315
                }
316
            }
317
            try {
318
                this.listWriter.writeHeader(values);
319
            } catch (IOException e) {
320
                if ( errorcounts++ <= 10 ) {
321
                    this.lasterror = e;
322
                    logger.warn("Can't write values to CVS file '" + getFullFileName() + "'.", e);
323
                    if ( errorcounts == 10 ) {
324
                        logger.warn("Too many error writing CVS file '" + getFullFileName() + "', don't output more.");
325
                    }
326
                }
327
            }
328

    
329
        }
330

    
331
        public void end() throws PerformEditingException {
332
            if ( this.errorcounts > 0 ) {
333
                throw new PerformEditingException(this.file.getAbsolutePath(), lasterror);
334
            }
335
            if ( listWriter != null ) {
336
                try {
337
                    listWriter.close();
338
                } catch (Exception ex) {
339
                    // Ignore error
340
                }
341
                listWriter = null;
342
            }
343
            if ( fwriter != null ) {
344
                try {
345
                    fwriter.close();
346
                } catch (Exception ex) {
347
                    // Ignore error
348
                }
349
                fwriter = null;
350
            }
351
        }
352

    
353
        public Envelope getEnvelope() {
354
            return this.envelope;
355
        }
356
    }
357

    
358
    public void performChanges(Iterator deleteds, Iterator inserteds, Iterator updateds, Iterator originalFeatureTypesUpdated) throws PerformEditingException {
359

    
360
        try {
361
            this.taskStatus.add();
362
            taskStatus.message("_preparing");
363
            getResource().execute(new ResourceAction() {
364
                public Object run() throws Exception {
365
                    FeatureSet features = null;
366
                    DisposableIterator it = null;
367
                    try {
368
                        File file = (File) resource.get();
369

    
370
                        Writer writer = new Writer();
371
                        writer.initialize(file, getStoreServices().getDefaultFeatureType(), getCSVPreferences());
372
                        features
373
                                = getStoreServices().getFeatureStore()
374
                                .getFeatureSet();
375
                        List<FeatureProvider> newdata = new ArrayList<FeatureProvider>();
376
                        writer.begin();
377
                        it = features.fastIterator();
378
                        taskStatus.setRangeOfValues(0, 0);
379
                        long counter = 0;
380
                        while ( it.hasNext() ) {
381
                            taskStatus.setCurValue(counter++);
382
                            FeatureProvider feature = getStoreServices().getFeatureProviderFromFeature(
383
                                    (org.gvsig.fmap.dal.feature.Feature) it.next());
384
                            writer.add(feature);
385
                            if ( feature.getOID() == null ) {
386
                                logger.warn("feature without OID");
387
                                feature.setOID(createNewOID());
388
                            }
389
                            newdata.add(feature);
390
                        }
391
                        data = newdata;
392
                        if ( writer.getEnvelope() != null ) {
393
                            envelope = writer.getEnvelope().getGeometry().getEnvelope();
394
                        }
395
                        resource.notifyChanges();
396
                        writer.end();
397
                    } finally {
398
                        if ( it != null ) {
399
                            it.dispose();
400
                        }
401
                        if ( features != null ) {
402
                            features.dispose();
403
                        }
404
                    }
405
                    return null;
406
                }
407

    
408
                private CsvPreference getCSVPreferences() {
409
                    CSVReader reader = new CSVReader(getCSVParameters());
410
                    return reader.getCSVPreferences();
411
                }
412
                
413
            });
414
            this.taskStatus.terminate();
415
        } catch (Exception e) {
416
            this.taskStatus.abort();
417
            throw new PerformEditingException(getResource().toString(), e);
418
        } finally {
419
            this.taskStatus.remove();
420
        }
421
    }
422

    
423
    public boolean closeResourceRequested(ResourceProvider resource) {
424
        return true;
425
    }
426

    
427
    public int getOIDType() {
428
        return DataTypes.LONG;
429
    }
430

    
431
    public boolean supportsAppendMode() {
432
        return false;
433
    }
434

    
435
    public void append(FeatureProvider featureProvider) {
436
        throw new UnsupportedOperationException();
437
    }
438

    
439
    public void beginAppend() {
440
        throw new UnsupportedOperationException();
441
    }
442

    
443
    public void endAppend() {
444
        throw new UnsupportedOperationException();
445
    }
446

    
447
    public void saveToState(PersistentState state) throws PersistenceException {
448
        throw new NotYetImplemented();
449
    }
450

    
451
    public void loadFromState(PersistentState state) throws PersistenceException {
452
        throw new NotYetImplemented();
453
    }
454

    
455
    public Object createNewOID() {
456
        return new Long(counterNewsOIDs++);
457
    }
458

    
459
    protected void initializeFeatureTypes() throws InitializeException {
460
        try {
461
            this.open();
462
        } catch (OpenException e) {
463
            throw new InitializeException(this.getProviderName(), e);
464
        }
465
    }
466

    
467
    public Envelope getEnvelope() throws DataException {
468
        this.open();
469
        if ( this.envelope != null ) {
470
            return this.envelope;
471
        }
472
        if ( !this.need_calculate_envelope ) {
473
            return null;
474
        }
475
        FeatureStore fs = this.getFeatureStore();
476
        FeatureType ft = fs.getDefaultFeatureType();
477
        FeatureAttributeDescriptor fad = ft.getAttributeDescriptor(ft.getDefaultGeometryAttributeIndex());
478

    
479
        try {
480
            this.envelope = GeometryLocator.getGeometryManager().createEnvelope(fad.getGeomType().getSubType());
481
            fs.accept(new Visitor() {
482
                public void visit(Object obj) throws VisitCanceledException, BaseException {
483
                    Feature f = (Feature) obj;
484
                    Geometry geom = f.getDefaultGeometry();
485
                    if ( geom != null ) {
486
                        envelope.add(geom.getEnvelope());
487
                    }
488
                }
489
            });
490
        } catch (BaseException e) {
491
            logger.warn("Can't calculate the envelope of CSV file '" + this.getFullName() + "'.", e);
492
            this.envelope = null;
493
        }
494

    
495
        this.need_calculate_envelope = false;
496
        return this.envelope;
497
    }
498

    
499
    public Object getDynValue(String name) throws DynFieldNotFoundException {
500
        if ( DataStore.METADATA_ENVELOPE.equalsIgnoreCase(name) ) {
501
            try {
502
                return this.getEnvelope();
503
            } catch (DataException e) {
504
                return null;
505
            }
506
        } else {
507
            if ( DataStore.METADATA_CRS.equalsIgnoreCase(name) ) {
508
                IProjection pro = CSVStoreParameters.getCRS(this.getCSVParameters());
509
                if ( pro != null ) {
510
                    return pro;
511
                }
512
            }
513
        }
514
        return super.getDynValue(name);
515
    }
516

    
517
    public void resourceChanged(ResourceProvider resource) {
518
        this.getStoreServices().notifyChange(
519
                DataStoreNotification.RESOURCE_CHANGED,
520
                resource);
521
    }
522

    
523
    public Object getSourceId() {
524
        return this.getCSVParameters().getFile();
525
    }
526

    
527
    public String getName() {
528
        String name = this.getCSVParameters().getFile().getName();
529
        return FilenameUtils.getBaseName(name);
530
    }
531

    
532
    public String getFullName() {
533
        return this.getCSVParameters().getFile().getAbsolutePath();
534
    }
535

    
536
    public ResourceProvider getResource() {
537
        return resource;
538
    }
539

    
540
    private boolean isEmpty(String s) {
541
        if ( s == null ) {
542
            return true;
543
        }
544
        return s.trim().length() == 0;
545
    }
546

    
547
    private class FieldTypeParser {
548

    
549
        public String name = null;
550
        public int type = DataTypes.STRING;
551
        public int size = 0;
552
        public boolean allowNulls = true;
553

    
554
        private String typename = "string";
555

    
556
        FieldTypeParser() {
557
        }
558

    
559
        private int getType(String value) {
560
            DataTypesManager dataTypesManager = ToolsLocator.getDataTypesManager();
561
            return dataTypesManager.getType(typename);
562
        }
563

    
564
        // El formato seria:
565
        //   name[:typename[:size[:notnull|null]]]
566
        //   name[__typename[__size[__notnull|null]]]
567
        //
568
        public boolean parse(String value) {
569
            String typename = null;
570
            String[] ss = null;
571
            if ( value.contains(":") ) {
572
                ss = value.split(":");
573
            } else if ( value.contains("__") ) {
574
                ss = value.split("__");
575
            }
576
            if ( ss == null ) {
577
                this.name = value;
578
                return true;
579
            }
580
            switch (ss.length) {
581
            case 4:
582
                if ( ss[3].length() > 0 ) {
583
                    if ( "notnull".equalsIgnoreCase(ss[3]) ) {
584
                        this.allowNulls = false;
585
                    } else {
586
                        this.allowNulls = true;
587
                    }
588
                }
589
            case 3:
590
                if ( ss[2].length() > 0 ) {
591
                    try {
592
                        this.size = Integer.parseInt(ss[2]);
593
                    } catch (Exception ex) {
594
                        logger.warn("Ignore incorrect field size for field " + value + " in CSV header of '" + getFullFileName() + "'.", ex);
595
                    }
596
                }
597
            case 2:
598
                if ( ss[1].length() > 0 ) {
599
                    this.typename = ss[1];
600
                    this.type = this.getType(this.typename);
601
                    if ( this.type == DataTypes.INVALID ) {
602
                        this.type = DataTypes.STRING;
603
                        logger.info("Type '" + typename + "' not valid for attribute '" + value + "' in CSV file '" + getFullFileName() + "'.");
604
                    }
605
                }
606
            case 1:
607
                this.name = ss[0];
608
                break;
609
            }
610

    
611
            if ( this.type != DataTypes.STRING ) {
612
                this.size = 0;
613
            }
614
            return true;
615
        }
616

    
617
    }
618

    
619
    private EditableFeatureType getFeatureType(String headers[], int automaticTypes[]) {
620
        EditableFeatureType fType = getStoreServices().createFeatureType(this.getName());
621
        fType.setHasOID(true);
622
        DataTypesManager dataTypesManager = ToolsLocator.getDataTypesManager();
623

    
624
        FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length];
625
        //
626
        // Calculamos cuales pueden ser los tipos de datos
627
        //
628
        for ( int i = 0; i < fieldTypes.length; i++ ) {
629
            fieldTypes[i] = new FieldTypeParser();
630
        }
631

    
632
        // Asuminos los tipos pasados por parametro, que se supone
633
        // son los detectados automaticamente.
634
        if ( automaticTypes != null ) {
635
            for ( int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++ ) {
636
                fieldTypes[i].type = automaticTypes[i];
637
            }
638
        }
639
        // Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo
640
        // los tipos anteriores en caso de definirse en la cabezara.
641
        for ( int i = 0; i < fieldTypes.length; i++ ) {
642
            if ( !fieldTypes[i].parse(headers[i]) ) {
643
                continue;
644
            }
645

    
646
        }
647

    
648
        // Y por ultimo hacemos caso a lo que se haya especificado en los parametros
649
        // de apertura del CSV, teniendo esto prioridad sobre todo.
650
        int[] param_types = CSVStoreParameters.getFieldTypes(this.getParameters());
651
        if ( param_types != null ) {
652
            for ( int i = 0; i < fieldTypes.length && i < param_types.length; i++ ) {
653
                fieldTypes[i].type = param_types[i];
654
            }
655
        }
656

    
657
        int[] param_sizes = CSVStoreParameters.getFieldSizes(this.getParameters());
658
        if ( param_sizes != null ) {
659
            for ( int i = 0; i < param_sizes.length; i++ ) {
660
                if ( param_sizes[i] > 0 ) {
661
                    fieldTypes[i].size = param_sizes[i];
662
                }
663
            }
664
        }
665
        //
666
        // Una vez ya sabemos los tipos de datos rellenamos el feature-type
667
        //
668
        for ( int i = 0; i < fieldTypes.length; i++ ) {
669
            EditableFeatureAttributeDescriptor fad = fType.add(
670
                    fieldTypes[i].name,
671
                    fieldTypes[i].type
672
            );
673
            fad.setSize(fieldTypes[i].size);
674
            fad.setAllowNull(fieldTypes[i].allowNulls);
675
            if ( fieldTypes[i].type == DataTypes.GEOMETRY
676
                    && fType.getDefaultGeometryAttributeName() == null ) {
677
                fType.setDefaultGeometryAttributeName(fieldTypes[i].name);
678
            }
679
        }
680
        String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters());
681
        if ( pointDimensionNames != null ) {
682
//            ToPointEvaluaror evaluator = new ToPointEvaluaror(pointDimensionNames);
683
            PointAttributeEmulator emulator = new PointAttributeEmulator(pointDimensionNames);
684
            EditableFeatureAttributeDescriptor attr = fType.add("GEOM", DataTypes.GEOMETRY, emulator);
685
            GeometryManager geommgr = GeometryLocator.getGeometryManager();
686
            GeometryType gt;
687
            try {
688
                gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM3D);
689
                attr.setGeometryType(gt);
690
                attr.setGeometryType(Geometry.TYPES.POINT);
691
                attr.setGeometrySubType(Geometry.SUBTYPES.GEOM3D);                
692
            } catch (Exception e) {
693
                logger.warn("Can't set geometry type for the calculated field in CSV file '" + getFullFileName() + "'.", e);
694
            }
695
        }
696
        return fType;
697
    }
698

    
699
    static class PointAttributeEmulator implements FeatureAttributeEmulator {
700

    
701
        private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class);
702

    
703
        private static final int XNAME = 0;
704
        private static final int YNAME = 1;
705
        private static final int ZNAME = 2;
706

    
707
        private final GeometryManager geommgr;
708
        private final String[] fieldNames;
709
        private final Coercion toDouble;
710
        private final DataType dataType;
711
        private int errorcount = 0;
712

    
713
        public PointAttributeEmulator(String[] pointDimensionNames) {
714
            if ( pointDimensionNames.length > 2 ) {
715
                this.fieldNames = new String[3];
716
                this.fieldNames[ZNAME] = pointDimensionNames[2];
717
            } else {
718
                this.fieldNames = new String[2];
719
            }
720
            this.fieldNames[XNAME] = pointDimensionNames[0];
721
            this.fieldNames[YNAME] = pointDimensionNames[1];
722
            this.geommgr = GeometryLocator.getGeometryManager();
723
            DataTypesManager datatypeManager = ToolsLocator.getDataTypesManager();
724

    
725
            this.toDouble = datatypeManager.getCoercion(DataTypes.DOUBLE);
726
            this.dataType = datatypeManager.get(DataTypes.GEOMETRY);
727
        }
728

    
729
        public Object get(Feature feature) {
730
            try {
731
                Object valueX = feature.get(this.fieldNames[XNAME]);
732
                valueX = toDouble.coerce(valueX);
733
                if ( valueX == null ) {
734
                    return null;
735
                }
736
                Object valueY = feature.get(this.fieldNames[YNAME]);
737
                valueY = toDouble.coerce(valueY);
738
                if ( valueY == null ) {
739
                    return null;
740
                }
741
                Object valueZ = null;
742
                if ( this.fieldNames.length > 2 ) {
743
                    valueZ = toDouble.coerce(feature.get(this.fieldNames[ZNAME]));
744
                    if ( valueZ == null ) {
745
                        return null;
746
                    }
747
                }
748

    
749
                double x = ((Double) valueX).doubleValue();
750
                double y = ((Double) valueY).doubleValue();
751
                Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);
752
                if ( this.fieldNames.length > 2 ) {
753
                    double z = ((Double) valueZ).doubleValue();
754
                    point.setCoordinateAt(2, z);
755
                }
756
                return point;
757
            } catch (Exception ex) {
758
                if ( ++errorcount < 5 ) {
759
                    logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='"
760
                            + this.fieldNames[XNAME] + "', YNAME='" + this.fieldNames[XNAME] + "' feature=" + feature.toString(), ex);
761
                }
762
                return null;
763
            }
764
        }
765

    
766
        public void set(EditableFeature feature, Object value) {
767
            if ( value == null ) {
768
                return;
769
            }
770
            Point point = null;
771
            if ( value instanceof MultiPoint ) {
772
                point = (Point) ((MultiPoint) value).getPrimitiveAt(0);
773
            } else {
774
                point = (Point) value;
775
            }
776
            feature.set(this.fieldNames[XNAME], point.getX());
777
            feature.set(this.fieldNames[YNAME], point.getY());
778
            if ( this.fieldNames.length > 2 ) {
779
                feature.set(this.fieldNames[ZNAME], point.getCoordinateAt(2));
780
            }
781
        }
782

    
783
        public boolean allowSetting() {
784
            return true;
785
        }
786

    
787
        public String[] getRequiredFieldNames() {
788
            return this.fieldNames;
789
        }
790

    
791
    }
792

    
793
    static class ToPointEvaluaror extends AbstractEvaluator {
794

    
795
        private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class);
796

    
797
        private GeometryManager geommgr = null;
798
        private String xname = null;
799
        private String yname = null;
800
        private String zname = null;
801
        private Coercion toDouble;
802
        private int errorcount = 0;
803

    
804
        ToPointEvaluaror(String[] pointDimensionNames) {
805
            this.xname = pointDimensionNames[0];
806
            this.yname = pointDimensionNames[1];
807
            if ( pointDimensionNames.length > 2 ) {
808
                this.zname = pointDimensionNames[2];
809
            }
810
            this.geommgr = GeometryLocator.getGeometryManager();
811
            this.toDouble = ToolsLocator.getDataTypesManager().getCoercion(DataTypes.DOUBLE);
812
        }
813

    
814
        public Object evaluate(EvaluatorData data) throws EvaluatorException {
815
            try {
816
                double x = ((Double) toDouble.coerce(data.getDataValue(xname))).doubleValue();
817
                double y = ((Double) toDouble.coerce(data.getDataValue(yname))).doubleValue();
818
                Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);
819
                if ( zname != null ) {
820
                    double z = ((Double) toDouble.coerce(data.getDataValue(zname))).doubleValue();
821
                    point.setCoordinateAt(2, z);
822
                }
823
                return point;
824
            } catch (Exception ex) {
825
                if ( ++errorcount < 5 ) {
826
                    logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='"
827
                            + xname + "', YNAME='" + yname + "', ZNAME='" + zname + "', data=" + data.toString());
828
                }
829
                return null;
830
            }
831
        }
832

    
833
        public String getName() {
834
            return "ToPointEvaluaror";
835
        }
836

    
837
    }
838

    
839
    private SimpleReader getSimpleReader(FileReader in) {
840
        SimpleReader reader;
841
        if ( CSVStoreParameters.getRawFieldsDefinition(getCSVParameters()) != null ) {
842
            reader = new FixedLenReader(in, getCSVParameters());
843
        } else {
844
            reader = new CSVReader(in, getCSVParameters());
845
        }
846
        return reader;
847
    }
848

    
849
    private void loadFeatures() throws IOException, DataException,
850
            CoercionException, CloneNotSupportedException {
851
        FileReader in = null;
852
        SimpleReader reader = null;
853
        try {
854
            String headers[] = null;
855
            FeatureStoreProviderServices store = this.getStoreServices();
856

    
857
            boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());
858
            
859
            in = new FileReader(this.getCSVParameters().getFile());
860

    
861
            reader = getSimpleReader(in);
862

    
863
            headers = CSVStoreParameters.getHeaders(getCSVParameters());
864
            if ( headers == null ) {
865
                headers = reader.getHeader();
866
                if ( headers == null ) {
867
                    String msg = "Can't retrieve header from csv file '"
868
                            + this.getCSVParameters().getFile()
869
                            .getAbsolutePath()
870
                            + "' and not specified in the parameters.";
871
                    logger.warn(msg);
872
                    throw new RuntimeException(msg);
873
                }
874
            }
875

    
876
            // Initialize the feature types
877
            EditableFeatureType edftype = this.getFeatureType(headers, automaticDetectionOfTypes());
878
            FeatureType ftype = edftype.getNotEditableCopy();
879
            List<FeatureType> ftypes = new ArrayList<FeatureType>();
880
            ftypes.add(ftype);
881
            store.setFeatureTypes(ftypes, ftype);
882

    
883
            Coercion coercion[] = new Coercion[ftype.size()];
884
            int sizes[] = new int[ftype.size()];
885
            for ( int i = 0; i < ftype.size(); i++ ) {
886
                sizes[i] = -1;
887
                FeatureAttributeDescriptor ad = ftype.getAttributeDescriptor(i);
888
                coercion[i] = ad.getDataType().getCoercion();
889
                if ( ad.getDataType().getType() == DataTypes.STRING ) {
890
                    if ( ad.getSize() == 0 ) {
891
                        // Es un string y no tiene un size asignado.
892
                        // Lo ponemos a cero para calcularlo.
893
                        sizes[i] = 0;
894
                    }
895
                }
896
            }
897
            if ( ftype.getDefaultGeometryAttributeName() != null ) {
898
                this.need_calculate_envelope = true;
899
            }
900

    
901
            Locale locale = CSVStoreParameters.getLocale(getCSVParameters());
902
            taskStatus.message("_loading");
903
            int count = 0;
904

    
905
            int count_errors = 0;
906

    
907
            List<String> row = reader.read();
908

    
909
            int skipLines = CSVStoreParameters.getSkipLines(getCSVParameters());
910
            if ( skipLines > 0 ) {
911
                row = reader.skip(skipLines);
912
            }
913

    
914
            while ( row != null ) {
915
                taskStatus.setCurValue(++count);
916
                FeatureProvider feature = this.createFeatureProvider(ftype);
917
                for ( int i = 0; i < row.size(); i++ ) {
918
                    Object rawvalue = row.get(i);
919
                    try {
920
                        Object value = null;
921
                        if ( locale != null && coercion[i] instanceof CoercionWithLocale ) {
922
                            value = ((CoercionWithLocale) (coercion[i])).coerce(rawvalue, locale);
923
                        } else {
924
                            value = coercion[i].coerce(rawvalue);
925
                        }
926
                        feature.set(i, value);
927
                        if ( sizes[i] >= 0 && value != null ) {
928
                            int x = ((String) value).length();
929
                            if ( sizes[i] < x ) {
930
                                sizes[i] = x;
931
                            }
932
                        }
933
                    } catch (RuntimeException ex) {
934
                        if ( !ignore_errors ) {
935
                            throw ex;
936
                        }
937
                        if ( count_errors++ < 10 ) {
938
                            logger.warn("Can't load value of attribute " + i + " in row " + count + ".", ex);
939
                        }
940
                        if ( count_errors == 10 ) {
941
                            logger.info("Too many errors, suppress messages.");
942
                        }
943
                    }
944
                }
945
                this.addFeatureProvider(feature);
946
                row = reader.read();
947
            }
948
            for ( int i = 0; i < ftype.size(); i++ ) {
949
                if ( sizes[i] > 0 ) {
950
                    EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor) edftype.getAttributeDescriptor(i));
951
                    efad.setSize(sizes[i]);
952
                }
953
            }
954
            // Volvemos a asignar al store el featuretype, ya que puede
955
            // haber cambiado.
956
            ftype = edftype.getNotEditableCopy();
957
            ftypes = new ArrayList<FeatureType>();
958
            ftypes.add(ftype);
959
            store.setFeatureTypes(ftypes, ftype);
960

    
961
            taskStatus.terminate();
962
        } catch(Exception ex) {
963
            int lineno = 0;
964
            if( reader!=null ) {
965
                lineno = reader.getLine();
966
            }
967
            throw new RuntimeException("Problems reading file '"+getFullFileName()+"' near line "+lineno+".", ex);
968

    
969
        } finally {
970
            if ( reader != null ) {
971
                try {
972
                    reader.close();
973
                } catch (Exception ex) {
974
                    // Do nothing
975
                }
976
                reader = null;
977
            }
978
            if ( in != null ) {
979
                try {
980
                    in.close();
981
                } catch (Exception ex) {
982
                    // Do nothing
983
                }
984
                in = null;
985
            }
986
        }
987
    }
988

    
989
    private int[] automaticDetectionOfTypes() throws IOException {
990
        boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(getCSVParameters());
991
        if ( !automatic_types_detection ) {
992
            return null;
993
        }
994

    
995
        final int T_INT = 0;
996
        final int T_FLOAT = 1;
997
        final int T_DOUBLE = 2;
998
        final int T_LONG = 3;
999
        final int T_URL = 4;
1000
        final int T_DATE = 5;
1001
        boolean possibleDataTypes[][] = null;
1002
        Locale locale = null;
1003
        int[] types = null;
1004

    
1005
        FileReader in = null;
1006
        SimpleReader reader = null;
1007
        String headers[] = null;
1008
        SimpleDateFormat dateFormat = new SimpleDateFormat();
1009

    
1010
        try {
1011

    
1012
            in = new FileReader(this.getCSVParameters().getFile());
1013

    
1014
            reader = getSimpleReader(in);
1015
            headers = reader.getHeader();
1016
            if ( headers == null ) {
1017
                headers = CSVStoreParameters.getHeaders(getCSVParameters());
1018
            }
1019
            types = new int[headers.length];
1020

    
1021
            possibleDataTypes = new boolean[headers.length][6];
1022
            for ( int i = 0; i < possibleDataTypes.length; i++ ) {
1023
                for ( int j = 0; j < 4; j++ ) {
1024
                    possibleDataTypes[i][j] = true;
1025
                }
1026
            }
1027
            locale = CSVStoreParameters.getLocale(getCSVParameters());
1028
            if ( locale == null ) {
1029
                locale = Locale.getDefault();
1030
            }
1031

    
1032
            DataTypesManager typeManager = ToolsLocator.getDataTypesManager();
1033
            CoercionWithLocale toDouble = (CoercionWithLocale) typeManager.getCoercion(DataTypes.DOUBLE);
1034
            CoercionWithLocale toFloat = (CoercionWithLocale) typeManager.getCoercion(DataTypes.FLOAT);
1035

    
1036
            List<String> row = reader.read();
1037

    
1038
            while ( row != null ) {
1039
                for ( int i = 0; i < row.size(); i++ ) {
1040
                    Object rawvalue = row.get(i);
1041
                    Object value = null;
1042
                    if ( possibleDataTypes[i][T_DOUBLE] ) {
1043
                        try {
1044
                            value = toDouble.coerce(rawvalue, locale);
1045
                            possibleDataTypes[i][T_DOUBLE] = true;
1046
                        } catch (Exception ex) {
1047
                            possibleDataTypes[i][T_DOUBLE] = false;
1048
                        }
1049
                    }
1050
                    if ( possibleDataTypes[i][T_FLOAT] ) {
1051
                        try {
1052
                            value = toFloat.coerce(rawvalue, locale);
1053
                            possibleDataTypes[i][T_FLOAT] = true;
1054
                        } catch (Exception ex) {
1055
                            possibleDataTypes[i][T_FLOAT] = false;
1056
                        }
1057
                    }
1058
                    if ( possibleDataTypes[i][T_LONG] ) {
1059
                        try {
1060
                            value = Long.parseLong((String) rawvalue);
1061
                            possibleDataTypes[i][T_LONG] = true;
1062
                        } catch (Exception ex) {
1063
                            possibleDataTypes[i][T_LONG] = false;
1064
                        }
1065
                    }
1066
                    if ( possibleDataTypes[i][T_INT] ) {
1067
                        try {
1068
                            value = Integer.parseInt((String) rawvalue);
1069
                            possibleDataTypes[i][T_INT] = true;
1070
                        } catch (Exception ex) {
1071
                            possibleDataTypes[i][T_INT] = false;
1072
                        }
1073
                    }
1074
                    if ( possibleDataTypes[i][T_DATE] ) {
1075
                        try {
1076
                            value = dateFormat.parse((String) rawvalue);
1077
                            possibleDataTypes[i][T_DATE] = true;
1078
                        } catch (Exception ex) {
1079
                            possibleDataTypes[i][T_DATE] = false;
1080
                        }
1081
                    }
1082
                    if ( possibleDataTypes[i][T_URL] ) {
1083
                        try {
1084
                            value = new URL((String) rawvalue);
1085
                            possibleDataTypes[i][T_URL] = true;
1086
                        } catch (Exception ex) {
1087
                            possibleDataTypes[i][T_URL] = false;
1088
                        }
1089
                    }
1090
                }
1091
                row = reader.read();
1092
            }
1093
        } catch(Exception ex) {
1094
            int lineno = 0;
1095
            if( reader!=null ) {
1096
                lineno = reader.getLine();
1097
            }
1098
            throw new RuntimeException("Problems reading file '"+getFullFileName()+"' near line "+lineno+".", ex);
1099

    
1100
        } finally {
1101
            if ( reader != null ) {
1102
                try {
1103
                    reader.close();
1104
                } catch (Exception ex) {
1105
                    // Do nothing
1106
                }
1107
                reader = null;
1108
            }
1109
            if ( in != null ) {
1110
                try {
1111
                    in.close();
1112
                } catch (Exception ex) {
1113
                    // Do nothing
1114
                }
1115
                in = null;
1116
            }
1117

    
1118
        }
1119
        for ( int i = 0; i < possibleDataTypes.length; i++ ) {
1120
            if ( possibleDataTypes[i][T_INT] ) {
1121
                types[i] = DataTypes.INT;
1122
                continue;
1123
            }
1124
            if ( possibleDataTypes[i][T_LONG] ) {
1125
                types[i] = DataTypes.LONG;
1126
                continue;
1127
            }
1128
            if ( possibleDataTypes[i][T_FLOAT] ) {
1129
                types[i] = DataTypes.FLOAT;
1130
                continue;
1131
            }
1132
            if ( possibleDataTypes[i][T_DOUBLE] ) {
1133
                types[i] = DataTypes.DOUBLE;
1134
                continue;
1135
            }
1136
            if ( possibleDataTypes[i][T_URL] ) {
1137
                types[i] = DataTypes.URL;
1138
                continue;
1139
            }
1140
            if ( possibleDataTypes[i][T_DATE] ) {
1141
                types[i] = DataTypes.DATE;
1142
                continue;
1143
            }
1144
            types[i] = DataTypes.STRING;
1145
        }
1146
        return types;
1147
    }
1148

    
1149
}