Revision 45684 trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVStoreProvider.java

View differences:

CSVStoreProvider.java
39 39
import java.util.Map;
40 40
import javax.json.JsonArray;
41 41
import javax.json.JsonObject;
42
import javax.json.JsonValue;
43 42

  
44 43
import org.apache.commons.io.FilenameUtils;
45 44
import org.apache.commons.io.IOUtils;
......
56 55
import org.gvsig.fmap.dal.exception.InitializeException;
57 56
import org.gvsig.fmap.dal.exception.OpenException;
58 57
import org.gvsig.fmap.dal.exception.ReadException;
59
import org.gvsig.fmap.dal.exception.ValidateDataParametersException;
60 58
import org.gvsig.fmap.dal.feature.EditableFeature;
61 59
import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;
62 60
import org.gvsig.fmap.dal.feature.EditableFeatureType;
......
68 66
import org.gvsig.fmap.dal.feature.FeatureType;
69 67
import org.gvsig.fmap.dal.feature.exception.PerformEditingException;
70 68
import org.gvsig.fmap.dal.feature.spi.FeatureProvider;
71
import org.gvsig.fmap.dal.feature.spi.FeatureStoreProviderServices;
72 69
import org.gvsig.fmap.dal.feature.spi.memory.AbstractMemoryStoreProvider;
73 70
import org.gvsig.fmap.dal.resource.ResourceAction;
74 71
import org.gvsig.fmap.dal.resource.file.FileResource;
......
212 209
                    .createServerExplorerParameters(FilesystemServerExplorer.NAME);
213 210
            params.setRoot(this.getCSVParameters().getFile().getParent());
214 211
            return manager.openServerExplorer(FilesystemServerExplorer.NAME, params);
215
        } catch (DataException e) {
212
        } catch (Exception e) {
216 213
            throw new ReadException(this.getProviderName(), e);
217
        } catch (ValidateDataParametersException e) {
218
            throw new ReadException(this.getProviderName(), e);
219 214
        }
220 215

  
221 216
    }
222 217

  
223 218
    @Override
219
    @SuppressWarnings("Convert2Lambda")
224 220
    public void performChanges(Iterator deleteds, Iterator inserteds, Iterator updateds, Iterator originalFeatureTypesUpdated) throws PerformEditingException {
225 221

  
226 222
        try {
......
238 234
                        features
239 235
                                = getStoreServices().getFeatureStore()
240 236
                                        .getFeatureSet();
241
                        List<FeatureProvider> newdata = new ArrayList<FeatureProvider>();
237
                        List<FeatureProvider> newdata = new ArrayList<>();
242 238
                        FeatureType ftype = getStoreServices().getDefaultFeatureType();
243 239
                        writer.initialize(getCSVParameters(), file, ftype, getCSVPreferences());
244 240
                        writer.begin();
......
304 300
    }
305 301

  
306 302
    @Override
303
    @SuppressWarnings("Convert2Lambda")
307 304
    public void append(final FeatureProvider featureProvider) {
308 305
        //todo
309 306
        getResource().execute(new ResourceAction() {
......
317 314
    }
318 315

  
319 316
    @Override
317
    @SuppressWarnings("Convert2Lambda")
320 318
    public void beginAppend() throws DataException {
321 319
        this.close();
322 320
        getResource().execute(new ResourceAction() {
321
            @Override
323 322
            public Object run() throws Exception {
324 323
                writer.initialize(
325 324
                        getCSVParameters(),
......
335 334
    }
336 335

  
337 336
    @Override
337
    @SuppressWarnings("Convert2Lambda")
338 338
    public void endAppend() {
339 339
        try {
340 340
            getResource().execute(new ResourceAction() {
341
                @Override
341 342
                public Object run() throws Exception {
342 343
                    writer.end();
343 344
                    resource.notifyChanges(); //resourcesNotifyChanges();
......
373 374
    }
374 375

  
375 376
    @Override
377
    @SuppressWarnings("Convert2Lambda")
376 378
    public Envelope getEnvelope() throws DataException {
377 379
        this.open();
378 380
        if (this.envelope != null) {
......
618 620

  
619 621
    }
620 622

  
621
    private EditableFeatureType getFeatureType(String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) {
622
        EditableFeatureType fType = getStoreServices().createFeatureType(this.getName());
623
    private static void fillFeatureType(CSVStoreParameters parameters, EditableFeatureType fType, String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) {
624
        String fullFileName = parameters.getFile()==null? "":parameters.getFile().getAbsolutePath();
625
        String providerName = NAME;
626
        
623 627
        fType.setHasOID(true);
624 628

  
629

  
625 630
        FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length];
626 631
        //
627 632
        // Calculamos cuales pueden ser los tipos de datos
628 633
        //
629 634
        for (int i = 0; i < fieldTypes.length; i++) {
630
            fieldTypes[i] = new FieldTypeParser(getProviderName(), getFullFileName());
635
            fieldTypes[i] = new FieldTypeParser(providerName, fullFileName);
631 636
        }
632 637

  
633 638
        // Asuminos los tipos pasados por parametro, que se supone
......
642 647
        // los tipos anteriores en caso de definirse en la cabezara.
643 648
        for (int i = 0; i < fieldTypes.length; i++) {
644 649
            if (!fieldTypes[i].parse(headers[i])) {
645
                continue;
650
                LOGGER.warn("Can't parse header of field "+i+ "( "+headers[i]+") in '"+providerName+"' file '" + fullFileName + "'.");
646 651
            }
647 652

  
648 653
        }
649 654

  
650 655
        // Y por ultimo hacemos caso a lo que se haya especificado en los parametros
651 656
        // de apertura del CSV, teniendo esto prioridad sobre todo.
652
        String param_types_def = CSVStoreParameters.getRawFieldTypes(this.getParameters());
657
        String param_types_def = CSVStoreParameters.getRawFieldTypes(parameters);
653 658
        if (StringUtils.isNotBlank(param_types_def)) {
654 659
            String sep = CSVStoreParameters.getDelimiter(param_types_def);
655 660
            if (StringUtils.isNotBlank(sep)) {
656 661
                String[] param_types = param_types_def.split(sep);
657
                FieldTypeParser parser = new FieldTypeParser(getProviderName(), getFullFileName());
662
                FieldTypeParser parser = new FieldTypeParser(providerName, fullFileName);
658 663
                for (String param_type : param_types) {
659 664
                    parser.clear();
660 665
                    parser.parse(param_type);
......
690 695
                    fType.setDefaultGeometryAttributeName(fieldType.name);
691 696
                }
692 697
            }
693
            Locale locale = CSVStoreParameters.getLocale(this.getParameters());
698
            Locale locale = CSVStoreParameters.getLocale(parameters);
694 699
            fad.setLocale(locale);
695 700
            for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) {
696 701
                try {
......
723 728
                            break;
724 729
                    }
725 730
                } catch (Exception ex) {
726
                    LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+getFullFileName()+"'.", ex);
731
                    LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+fullFileName+"'.", ex);
727 732
                }
728 733
            }
729 734
        }
730
        String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters());
735
        String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(parameters);
731 736
        if ( pointDimensionNames != null ) {
732 737
            PointAttributeEmulator emulator = new PointAttributeEmulator(pointDimensionNames);
733
            String columnName = CSVStoreParameters.getPointColumnName(this.getParameters());
738
            String columnName = CSVStoreParameters.getPointColumnName(parameters);
734 739
            if( StringUtils.isBlank(columnName) ) {
735 740
                columnName = "the_geom";
736 741
            }
......
745 750
                }
746 751
                attr.setGeometryType(gt);
747 752
            } catch (Exception e) {
748
                LOGGER.warn("Can't set geometry type for the calculated field in '"+getProviderName()+"' file '" + getFullFileName() + "'.", e);
753
                LOGGER.warn("Can't set geometry type for the calculated field in '"+providerName+"' file '" + fullFileName + "'.", e);
749 754
            }
750 755
        }        
751 756
        
752
        String geometry_column = CSVStoreParameters.getGeometryColumn(this.getParameters());
757
        String geometry_column = CSVStoreParameters.getGeometryColumn(parameters);
753 758
        if (!StringUtils.isEmpty(geometry_column)) {
754 759
            EditableFeatureAttributeDescriptor attr = (EditableFeatureAttributeDescriptor) fType.get(geometry_column);
755 760
            if (attr != null && attr.getType() != DataTypes.GEOMETRY) {
......
760 765
                    gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.UNKNOWN);
761 766
                    attr.setGeometryType(gt);
762 767
                } catch (Exception e) {
763
                    LOGGER.warn("Can't set geometry type for the calculated field in CSV file '" + getFullFileName() + "'.", e);
768
                    LOGGER.warn("Can't set geometry type for the calculated field in CSV file '" + fullFileName + "'.", e);
764 769
                }
765 770
                fType.setDefaultGeometryAttributeName(geometry_column);
766 771
            }
767 772
        }
768
        return fType;
769 773
    }
770 774

  
771 775
    static class PointAttributeEmulator implements FeatureAttributeEmulator {
......
840 844
            }
841 845
        }
842 846

  
847
        @Override
843 848
        public void set(EditableFeature feature, Object value) {
844 849
            if (value == null) {
845 850
                return;
846 851
            }
847
            Point point = null;
852
            Point point;
848 853
            if (value instanceof MultiPoint) {
849 854
                point = (Point) ((MultiPoint) value).getPrimitiveAt(0);
850 855
            } else {
......
904 909
        private String xname = null;
905 910
        private String yname = null;
906 911
        private String zname = null;
907
        private Coercion toDouble;
912
        private final Coercion toDouble;
908 913
        private int errorcount = 0;
909 914

  
910 915
        ToPointEvaluaror(String[] pointDimensionNames) {
......
920 925
        @Override
921 926
        public Object evaluate(EvaluatorData data) throws EvaluatorException {
922 927
            try {
923
                double x = ((Double) toDouble.coerce(data.getDataValue(xname))).doubleValue();
924
                double y = ((Double) toDouble.coerce(data.getDataValue(yname))).doubleValue();
928
                double x = ((Double) toDouble.coerce(data.getDataValue(xname)));
929
                double y = ((Double) toDouble.coerce(data.getDataValue(yname)));
925 930
                Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);
926 931
                if (zname != null) {
927
                    double z = ((Double) toDouble.coerce(data.getDataValue(zname))).doubleValue();
932
                    double z = ((Double) toDouble.coerce(data.getDataValue(zname)));
928 933
                    point.setCoordinateAt(2, z);
929 934
                }
930 935
                return point;
......
944 949

  
945 950
    }
946 951

  
947
    private SimpleReader getSimpleReader(InputStreamReader in) throws IOException {
952
    private static SimpleReader getSimpleReader(CSVStoreParameters parameters, InputStreamReader in) throws IOException {
948 953
        SimpleReader reader;
949
        String filename = CSVStoreParameters.getFileName(getCSVParameters());
954
        String filename = CSVStoreParameters.getFileName(parameters);
950 955
        if (FilenameUtils.isExtension(filename, "json")){
951 956
//            reader= new JSonReader(in);
952
            reader= new JSonReader(in,getCSVParameters());
953
        } else if (CSVStoreParameters.getRawFieldsDefinition(getCSVParameters()) != null) {
954
            reader = new FixedLenReader(in, getCSVParameters());
957
            reader= new JSonReader(in,parameters);
958
        } else if (CSVStoreParameters.getRawFieldsDefinition(parameters) != null) {
959
            reader = new FixedLenReader(in, parameters);
955 960
        } else {
956
            reader = new CSVReader(in, getCSVParameters());
961
            reader = new CSVReader(in, parameters);
957 962
        }
958 963
        return reader;
959 964
    }
960 965

  
961
    private String getFixedHeader(int column) {
966
    private static String getFixedHeader(int column) {
962 967
        char[] header = new char[3];
963 968

  
964 969
        String s = String.format("%03d", column);
......
968 973
        return String.valueOf(header);
969 974
    }
970 975

  
971
    private String[] getFixedHeaders(int count) {
976
    private static String[] getFixedHeaders(int count) {
972 977
        String[] headers = new String[count];
973 978
        for (int i = 0; i < headers.length; i++) {
974 979
            headers[i] = getFixedHeader(i);
......
976 981
        return headers;
977 982
    }
978 983

  
979
    private InputStreamReader openFile(File f, String charsetName) throws FileNotFoundException {
984
    private static InputStreamReader openFile(File f, String charsetName) throws FileNotFoundException {
985
        String fullFileName = f==null? "NULL":f.getAbsolutePath();
980 986
        Charset charset = Charset.defaultCharset();
981 987
        FileInputStream fis = new FileInputStream(f);
982 988
        if (!StringUtils.isEmpty(charsetName)) {
......
984 990
                try {
985 991
                    charset = Charset.forName(charsetName);
986 992
                } catch (Throwable th) {
987
                    LOGGER.warn("Can't use charset '" + charsetName + "' for read csv '" + this.getFullFileName() + "'.", th);
993
                    LOGGER.warn("Can't use charset '" + charsetName + "' for read csv '" + fullFileName + "'.", th);
988 994
                }
989 995
            } else {
990
                LOGGER.warn("charset '" + charsetName + "' not supported for read csv '" + this.getFullFileName() + "'.");
996
                LOGGER.warn("charset '" + charsetName + "' not supported for read csv '" + fullFileName + "'.");
991 997
            }
992 998
        }
993 999
        InputStreamReader isr = new InputStreamReader(fis, charset);
994 1000
        return isr;
995 1001
    }
996 1002

  
997
    private void loadFeatures() {
1003
    public static void loadFeatureType(CSVStoreParameters parameters, EditableFeatureType featureType, boolean  detectTypes) throws IOException {
998 1004
        InputStreamReader in = null;
999 1005
        SimpleReader reader = null;
1000 1006
        try {
1001
            String headers[] = null;
1002
            FeatureStoreProviderServices store = this.getStoreServices();
1007
            String headers[];
1008
//            FeatureStoreProviderServices store = this.getStoreServices();
1003 1009

  
1004
            boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());
1010
            boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(parameters);
1005 1011

  
1006 1012
            in = openFile(
1007
                    this.getCSVParameters().getFile(),
1008
                    CSVStoreParameters.getCharset(this.getCSVParameters())
1013
                    parameters.getFile(),
1014
                    CSVStoreParameters.getCharset(parameters)
1009 1015
            );
1010 1016

  
1011
            reader = getSimpleReader(in);
1017
            reader = getSimpleReader(parameters, in);
1012 1018

  
1013
            headers = CSVStoreParameters.getHeaders(getCSVParameters());
1019
            headers = CSVStoreParameters.getHeaders(parameters);
1014 1020
            if (headers == null) {
1015
                if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
1021
                if (CSVStoreParameters.isFirstLineHeader(parameters)) {
1016 1022
                    headers = reader.getHeader();
1017 1023
                    if (headers == null) {
1018
                        if (CSVStoreParameters.getIgnoreErrors(getCSVParameters())) {
1024
                        if (CSVStoreParameters.getIgnoreErrors(parameters)) {
1019 1025
                            headers = getFixedHeaders(reader.getColumnsCount());
1020 1026
                        } else {
1021 1027
                            String msg = "Can't retrieve header from csv file '"
1022
                                    + this.getCSVParameters().getFile()
1028
                                    + parameters.getFile()
1023 1029
                                            .getAbsolutePath()
1024 1030
                                    + "' and not specified in the parameters.";
1025 1031
                            LOGGER.warn(msg);
......
1030 1036
                    headers = getFixedHeaders(reader.getColumnsCount());
1031 1037
                }
1032 1038
            } else {
1033
                if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
1039
                if (CSVStoreParameters.isFirstLineHeader(parameters)) {
1034 1040
                    reader.getHeader(); // Skip and ignore the header of file
1035 1041
                }
1036 1042
            }
1037 1043

  
1038
            AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = automaticDetectionOfTypes(headers);
1044
            AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = null;
1045
            if( detectTypes ) {
1046
                detectedTypes = automaticDetectionOfTypes(parameters, headers);
1047
            }
1039 1048
            if (detectedTypes != null && detectedTypes.length > headers.length) {
1040 1049
                // Se han detectado mas columnas que las que hay en la cabezera,
1041 1050
                // a?adimos mas columnas a la cabezera.
......
1055 1064
                }
1056 1065
            }
1057 1066
            // Initialize the feature types
1058
            EditableFeatureType edftype = this.getFeatureType(headers, detectedTypes);
1067
            fillFeatureType(parameters, featureType,  headers, detectedTypes);
1068
        } finally {
1069
            IOUtils.closeQuietly(in);
1070
            IOUtils.closeQuietly(reader);
1071
        }
1072
    }
1073
    
1074
    private void loadFeatures() {
1075
        InputStreamReader in = null;
1076
        SimpleReader reader = null;
1077
        try {
1078
            boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());
1079

  
1080
            // Initialize the feature types
1081
            EditableFeatureType edftype = getStoreServices().createFeatureType(this.getName());
1082
            loadFeatureType(getCSVParameters(), edftype, true);
1059 1083
            FeatureType ftype = edftype.getNotEditableCopy();
1060 1084
            this.setFeatureType(ftype);
1061 1085

  
1086
            in = openFile(
1087
                    this.getCSVParameters().getFile(),
1088
                    CSVStoreParameters.getCharset(this.getCSVParameters())
1089
            );
1090
            reader = getSimpleReader(getCSVParameters(), in);
1091
            if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
1092
                reader.getHeader(); // Skip and ignore the header of file
1093
            }
1062 1094
            Coercion coercion[] = new Coercion[ftype.size()];
1063 1095
            CoercionContext coercionContext[] = new CoercionContext[ftype.size()];
1064 1096
            int sizes[] = new int[ftype.size()];
......
1170 1202
                } catch (Exception ex) {
1171 1203
                    // Do nothing
1172 1204
                }
1173
                reader = null;
1205
//                reader = null;
1174 1206
            }
1175 1207
            if (in != null) {
1176 1208
                try {
......
1178 1210
                } catch (Exception ex) {
1179 1211
                    // Do nothing
1180 1212
                }
1181
                in = null;
1213
//                in = null;
1182 1214
            }
1183 1215
        }
1184 1216
    }
1185 1217

  
1186
    private AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(String[] headers) throws IOException {
1187
        boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(getCSVParameters());
1218
    private static AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(CSVStoreParameters parameters, String[] headers) throws IOException {
1219
        String fullFileName = parameters.getFile()==null? "NULL":parameters.getFile().getAbsolutePath();
1220
        boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(parameters);
1188 1221
        if (!automatic_types_detection) {
1189 1222
            return null;
1190 1223
        }
......
1194 1227
        SimpleReader reader = null;
1195 1228

  
1196 1229
        try {
1197
            in = new FileReader(this.getCSVParameters().getFile());
1198
            reader = getSimpleReader(in);
1230
            in = new FileReader(parameters.getFile());
1231
            reader = getSimpleReader(parameters, in);
1199 1232
            AutomaticDetectionOfTypes x = new AutomaticDetectionOfTypes(
1200
                    this.getFullFileName()
1233
                    fullFileName
1201 1234
            );
1202 1235
            types = x.detect(
1203 1236
                    headers.length,
1204 1237
                    reader,
1205
                    CSVStoreParameters.isFirstLineHeader(getCSVParameters()),
1206
                    CSVStoreParameters.getLocale(getCSVParameters())
1238
                    CSVStoreParameters.isFirstLineHeader(parameters),
1239
                    CSVStoreParameters.getLocale(parameters)
1207 1240
            );
1208 1241
        } catch (Exception ex) {
1209 1242
            int lineno = 0;
1210 1243
            if (reader != null) {
1211 1244
                lineno = reader.getLine();
1212 1245
            }
1213
            throw new RuntimeException("Problems reading file '" + getFullFileName() + "' near line " + lineno + ".", ex);
1246
            throw new RuntimeException("Problems reading file '" + fullFileName + "' near line " + lineno + ".", ex);
1214 1247

  
1215 1248
        } finally {
1216 1249
            IOUtils.closeQuietly(reader);
......
1232 1265
    }
1233 1266
    
1234 1267
    private void setFeatureType(FeatureType ftype) {
1235
        FeatureStoreProviderServices store = this.getStoreServices();
1236 1268
        List<FeatureType> ftypes = new ArrayList<>();
1237 1269
        ftypes.add(ftype);
1238 1270
        this.featureType = ftype;
1239
        store.setFeatureTypes(ftypes, ftype);
1271
        this.getStoreServices().setFeatureTypes(ftypes, ftype);
1240 1272
    }
1241 1273

  
1242 1274
}

Also available in: Unified diff