Application: gvSIG desktop

/**

 * gvSIG. Desktop Geographic Information System.

 *

 * Copyright (C) 2007-2013 gvSIG Association.

 *

 * This program is free software; you can redistribute it and/or modify it under

 * the terms of the GNU General Public License as published by the Free Software

 * Foundation; either version 3 of the License, or (at your option) any later

 * version.

 *

 * This program is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more

 * details.

 *

 * You should have received a copy of the GNU General Public License along with

 * this program; if not, write to the Free Software Foundation, Inc., 51

 * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

 *

 * For any additional information, do not hesitate to contact us at info AT

 * gvsig.com, or visit our website www.gvsig.com.

 */

package org.gvsig.fmap.dal.store.csv;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStreamReader;

import java.net.URI;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Iterator;

import java.util.List;

import java.util.Locale;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.cresques.cts.IProjection;

import org.gvsig.fmap.dal.DALLocator;

import org.gvsig.fmap.dal.DataManager;

import org.gvsig.fmap.dal.DataServerExplorer;

import org.gvsig.fmap.dal.DataStore;

import org.gvsig.fmap.dal.DataStoreNotification;

import org.gvsig.fmap.dal.DataTypes;

import org.gvsig.fmap.dal.FileHelper;

import org.gvsig.fmap.dal.exception.DataException;

import org.gvsig.fmap.dal.exception.InitializeException;

import org.gvsig.fmap.dal.exception.OpenException;

import org.gvsig.fmap.dal.exception.ReadException;

import org.gvsig.fmap.dal.exception.ValidateDataParametersException;

import org.gvsig.fmap.dal.feature.EditableFeature;

import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.EditableFeatureType;

import org.gvsig.fmap.dal.feature.Feature;

import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.FeatureAttributeEmulator;

import org.gvsig.fmap.dal.feature.FeatureSet;

import org.gvsig.fmap.dal.feature.FeatureStore;

import org.gvsig.fmap.dal.feature.FeatureType;

import org.gvsig.fmap.dal.feature.exception.PerformEditingException;

import org.gvsig.fmap.dal.feature.spi.FeatureProvider;

import org.gvsig.fmap.dal.feature.spi.FeatureStoreProviderServices;

import org.gvsig.fmap.dal.feature.spi.memory.AbstractMemoryStoreProvider;

import org.gvsig.fmap.dal.resource.ResourceAction;

import org.gvsig.fmap.dal.resource.file.FileResource;

import org.gvsig.fmap.dal.resource.spi.ResourceConsumer;

import org.gvsig.fmap.dal.resource.spi.ResourceProvider;

import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorer;

import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorerParameters;

import org.gvsig.fmap.dal.spi.DataStoreProviderServices;

import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReader;

import org.gvsig.fmap.dal.store.csv.simplereaders.FixedLenReader;

import org.gvsig.fmap.dal.store.csv.simplereaders.SimpleReader;

import org.gvsig.fmap.geom.Geometry;

import org.gvsig.fmap.geom.GeometryLocator;

import org.gvsig.fmap.geom.GeometryManager;

import org.gvsig.fmap.geom.aggregate.MultiPoint;

import org.gvsig.fmap.geom.primitive.Envelope;

import org.gvsig.fmap.geom.primitive.Point;

import org.gvsig.fmap.geom.type.GeometryType;

import org.gvsig.tools.ToolsLocator;

import org.gvsig.tools.dataTypes.CoercionException;

import org.gvsig.tools.dataTypes.DataType;

import org.gvsig.tools.dataTypes.DataTypesManager;

import org.gvsig.tools.dataTypes.DataTypesManager.Coercion;

import org.gvsig.tools.dataTypes.DataTypesManager.CoercionWithLocale;

import org.gvsig.tools.dispose.DisposableIterator;

import org.gvsig.tools.dynobject.exception.DynFieldNotFoundException;

import org.gvsig.tools.evaluator.AbstractEvaluator;

import org.gvsig.tools.evaluator.EvaluatorData;

import org.gvsig.tools.evaluator.EvaluatorException;

import org.gvsig.tools.exception.BaseException;

import org.gvsig.tools.exception.NotYetImplemented;

import org.gvsig.tools.persistence.PersistentState;

import org.gvsig.tools.persistence.exception.PersistenceException;

import org.gvsig.tools.task.SimpleTaskStatus;

import org.gvsig.tools.task.TaskStatusManager;

import org.gvsig.tools.visitor.VisitCanceledException;

import org.gvsig.tools.visitor.Visitor;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.supercsv.io.CsvListWriter;

import org.supercsv.prefs.CsvPreference;

public class CSVStoreProvider extends AbstractMemoryStoreProvider implements

        ResourceConsumer {

    private static final Logger logger = LoggerFactory.getLogger(CSVStoreProvider.class);

    public static final String NAME = "CSV";

    public static final String DESCRIPTION = "CSV file";

    public static final String METADATA_DEFINITION_NAME = NAME;

    private ResourceProvider resource;

    private long counterNewsOIDs = 0;

    private Envelope envelope;

    private boolean need_calculate_envelope = false;

    private SimpleTaskStatus taskStatus;

    public CSVStoreProvider(CSVStoreParameters parameters,

            DataStoreProviderServices storeServices) throws InitializeException {

        super(

                parameters,

                storeServices,

                FileHelper.newMetadataContainer(METADATA_DEFINITION_NAME)

        );

        TaskStatusManager manager = ToolsLocator.getTaskStatusManager();

        this.taskStatus = manager.createDefaultSimpleTaskStatus("CSV");

        counterNewsOIDs = 0;

        File file = getCSVParameters().getFile();

        resource = this.createResource(

                FileResource.NAME,

                new Object[]{file.getAbsolutePath()}

        );

        resource.addConsumer(this);

        initializeFeatureTypes();

    }

    private CSVStoreParameters getCSVParameters() {

        return (CSVStoreParameters) this.getParameters();

    }

    public String getProviderName() {

        return NAME;

    }

    public boolean allowWrite() {

        return false;

    }

    private String getFullFileName() {

        // Usar solo para mostrar mensajes en el logger.

        String s = "(unknow)";

        try {

            s = getCSVParameters().getFile().getAbsolutePath();

        } catch (Exception e2) {

            s = "(unknow)";

        }

        return s;

    }

    public void open() throws OpenException {

        if (this.data != null) {

            return;

        }

        this.data = new ArrayList<>();

        resource.setData(new HashMap());

        counterNewsOIDs = 0;

        try {

            loadFeatures();

        } catch (RuntimeException e) {

            logger.debug("Can't load features from CSV '" + getFullFileName() + "'.", e);

            throw e;

        } catch (Exception e) {

            logger.debug("Can't load features from CSV '" + getFullFileName() + "'.", e);

            throw new RuntimeException(e);

        }

    }

    public DataServerExplorer getExplorer() throws ReadException {

        DataManager manager = DALLocator.getDataManager();

        FilesystemServerExplorerParameters params;

        try {

            params = (FilesystemServerExplorerParameters) manager

                    .createServerExplorerParameters(FilesystemServerExplorer.NAME);

            params.setRoot(this.getCSVParameters().getFile().getParent());

            return manager.openServerExplorer(FilesystemServerExplorer.NAME, params);

        } catch (DataException e) {

            throw new ReadException(this.getProviderName(), e);

        } catch (ValidateDataParametersException e) {

            throw new ReadException(this.getProviderName(), e);

        }

    }

    class Writer {

        private Envelope envelope = null;

        private boolean calculate_envelope = false;

        private CsvListWriter listWriter = null;

        private CsvPreference csvpreferences = null;

        private FileWriter fwriter = null;

        private FeatureType ftype;

        private File file;

        private String[] values;

        private FeatureAttributeDescriptor[] descriptors;

        private Coercion convert = null;

        private int errorcounts = 0;

        private Throwable lasterror = null;

        private Locale locale = null;

        public void initialize(File file, FeatureType ftype, CsvPreference csvpreferences) {

            this.file = file;

            this.ftype = ftype;

            this.csvpreferences = csvpreferences;

            this.locale = CSVStoreParameters.getLocale(getCSVParameters());

            if (csvpreferences == null) {

                this.csvpreferences = CsvPreference.STANDARD_PREFERENCE;

            }

            if (ftype.getDefaultGeometryAttributeName() != null) {

                this.calculate_envelope = true;

            }

            this.descriptors = this.ftype.getAttributeDescriptors();

            this.convert = ToolsLocator.getDataTypesManager().getCoercion(org.gvsig.tools.dataTypes.DataTypes.STRING);

            this.errorcounts = 0;

        }

        public void begin() {

            try {

                this.fwriter = new FileWriter(file);

            } catch (IOException e) {

                logger.warn("Can't open file for write (" + file.getAbsolutePath() + ").", e);

                throw new RuntimeException(e);

            }

            this.listWriter = new CsvListWriter(this.fwriter, this.csvpreferences);

            int n = 0;

            for (int i = 0; i < descriptors.length; i++) {

                FeatureAttributeDescriptor descriptor = descriptors[i];

                if (descriptor.getEvaluator() == null) {

                    n++;

                }

            }

            String[] header = new String[n];

            this.values = new String[n];

            n = 0;

            for (int i = 0; i < descriptors.length; i++) {

                FeatureAttributeDescriptor descriptor = descriptors[i];

                if (descriptor.getEvaluator() == null) {

                    String name = descriptor.getName();

                    String typeName = descriptor.getDataTypeName();

                    if (descriptor.getDataType().getType() == DataTypes.STRING) {

                        header[n++] = name + "__" + typeName + "__" + descriptor.getSize();

                    } else {

                        header[n++] = name + "__" + typeName;

                    }

                }

            }

            try {

                listWriter.writeHeader(header);

            } catch (Exception e) {

                logger.warn("Can't write header '" + header.toString() + "' file for write (" + file.getAbsolutePath() + ").", e);

                throw new RuntimeException(e);

            }

        }

        public void add(FeatureProvider feature) {

            if (this.calculate_envelope) {

                Geometry geom = feature.getDefaultGeometry();

                if (geom != null) {

                    if (envelope == null) {

                        try {

                            envelope = (Envelope) geom.getEnvelope().clone();

                        } catch (CloneNotSupportedException e) {

                            logger.warn("Este error no deberia pasar, siempre se puede hacer un clone de un envelope.", e);

                        }

                    } else {

                        envelope.add(geom.getEnvelope());

                    }

                }

            }

            int n = 0;

            for (int i = 0; i < descriptors.length; i++) {

                FeatureAttributeDescriptor descriptor = descriptors[i];

                if (descriptor.getEvaluator() == null) {

                    Object value = feature.get(i);

                    try {

                        n++;

                        if (this.convert != null && this.convert instanceof CoercionWithLocale) {

                            values[n] = (String) ((CoercionWithLocale) this.convert).coerce(value, this.locale);

                        } else {

                            values[n] = (String) this.convert.coerce(value);

                        }

                    } catch (CoercionException e) {

                        try {

                            values[n] = value.toString();

                        } catch (Exception ex) {

                            values[n] = "";

                        }

                        if (errorcounts++ <= 10) {

                            this.lasterror = e;

                            logger.warn("Can't convert value of field " + i + " to string in CVS file '" + getFullFileName() + "'.", e);

                            if (errorcounts == 10) {

                                logger.warn("Too many error writing CVS file '" + getFullFileName() + "', don't output more.");

                            }

                        }

                    }

                }

            }

            try {

                this.listWriter.writeHeader(values);

            } catch (IOException e) {

                if (errorcounts++ <= 10) {

                    this.lasterror = e;

                    logger.warn("Can't write values to CVS file '" + getFullFileName() + "'.", e);

                    if (errorcounts == 10) {

                        logger.warn("Too many error writing CVS file '" + getFullFileName() + "', don't output more.");

                    }

                }

            }

        }

        public void end() throws PerformEditingException {

            if (this.errorcounts > 0) {

                throw new PerformEditingException(this.file.getAbsolutePath(), lasterror);

            }

            if (listWriter != null) {

                try {

                    listWriter.close();

                } catch (Exception ex) {

                    // Ignore error

                }

                listWriter = null;

            }

            if (fwriter != null) {

                try {

                    fwriter.close();

                } catch (Exception ex) {

                    // Ignore error

                }

                fwriter = null;

            }

        }

        public Envelope getEnvelope() {

            return this.envelope;

        }

    }

    public void performChanges(Iterator deleteds, Iterator inserteds, Iterator updateds, Iterator originalFeatureTypesUpdated) throws PerformEditingException {

        try {

            this.taskStatus.add();

            taskStatus.message("_preparing");

            getResource().execute(new ResourceAction() {

                public Object run() throws Exception {

                    FeatureSet features = null;

                    DisposableIterator it = null;

                    try {

                        File file = (File) resource.get();

                        Writer writer = new Writer();

                        writer.initialize(file, getStoreServices().getDefaultFeatureType(), getCSVPreferences());

                        features

                                = getStoreServices().getFeatureStore()

                                .getFeatureSet();

                        List<FeatureProvider> newdata = new ArrayList<FeatureProvider>();

                        writer.begin();

                        it = features.fastIterator();

                        taskStatus.setRangeOfValues(0, 0);

                        long counter = 0;

                        while (it.hasNext()) {

                            taskStatus.setCurValue(counter++);

                            FeatureProvider feature = getStoreServices().getFeatureProviderFromFeature(

                                    (org.gvsig.fmap.dal.feature.Feature) it.next());

                            writer.add(feature);

                            if (feature.getOID() == null) {

                                logger.warn("feature without OID");

                                feature.setOID(createNewOID());

                            }

                            newdata.add(feature);

                        }

                        data = newdata;

                        if (writer.getEnvelope() != null) {

                            envelope = writer.getEnvelope().getGeometry().getEnvelope();

                        }

                        resource.notifyChanges();

                        writer.end();

                    } finally {

                        if (it != null) {

                            it.dispose();

                        }

                        if (features != null) {

                            features.dispose();

                        }

                    }

                    return null;

                }

                private CsvPreference getCSVPreferences() {

                    CSVReader reader = new CSVReader(getCSVParameters());

                    return reader.getCSVPreferences();

                }

            });

            this.taskStatus.terminate();

        } catch (Exception e) {

            this.taskStatus.abort();

            throw new PerformEditingException(getResource().toString(), e);

        } finally {

            this.taskStatus.remove();

        }

    }

    public boolean closeResourceRequested(ResourceProvider resource) {

        return true;

    }

    public int getOIDType() {

        return DataTypes.LONG;

    }

    public boolean supportsAppendMode() {

        return false;

    }

    public void append(FeatureProvider featureProvider) {

        throw new UnsupportedOperationException();

    }

    public void beginAppend() {

        throw new UnsupportedOperationException();

    }

    public void endAppend() {

        throw new UnsupportedOperationException();

    }

    public void saveToState(PersistentState state) throws PersistenceException {

        throw new NotYetImplemented();

    }

    public void loadFromState(PersistentState state) throws PersistenceException {

        throw new NotYetImplemented();

    }

    public Object createNewOID() {

        return new Long(counterNewsOIDs++);

    }

    protected void initializeFeatureTypes() throws InitializeException {

        try {

            this.open();

        } catch (OpenException e) {

            throw new InitializeException(this.getProviderName(), e);

        }

    }

    public Envelope getEnvelope() throws DataException {

        this.open();

        if (this.envelope != null) {

            return this.envelope;

        }

        if (!this.need_calculate_envelope) {

            return null;

        }

        FeatureStore fs = this.getFeatureStore();

        FeatureType ft = fs.getDefaultFeatureType();

        FeatureAttributeDescriptor fad = ft.getAttributeDescriptor(ft.getDefaultGeometryAttributeIndex());

        try {

            this.envelope = GeometryLocator.getGeometryManager().createEnvelope(fad.getGeomType().getSubType());

            fs.accept(new Visitor() {

                public void visit(Object obj) throws VisitCanceledException, BaseException {

                    Feature f = (Feature) obj;

                    Geometry geom = f.getDefaultGeometry();

                    if (geom != null) {

                        envelope.add(geom.getEnvelope());

                    }

                }

            });

        } catch (BaseException e) {

            logger.warn("Can't calculate the envelope of CSV file '" + this.getFullName() + "'.", e);

            this.envelope = null;

        }

        this.need_calculate_envelope = false;

        return this.envelope;

    }

    public Object getDynValue(String name) throws DynFieldNotFoundException {

        if (DataStore.METADATA_ENVELOPE.equalsIgnoreCase(name)) {

            try {

                return this.getEnvelope();

            } catch (DataException e) {

                return null;

            }

        } else {

            if (DataStore.METADATA_CRS.equalsIgnoreCase(name)) {

                IProjection pro = CSVStoreParameters.getCRS(this.getCSVParameters());

                if (pro != null) {

                    return pro;

                }

            }

        }

        return super.getDynValue(name);

    }

    public void resourceChanged(ResourceProvider resource) {

        this.getStoreServices().notifyChange(

                DataStoreNotification.RESOURCE_CHANGED,

                resource);

    }

    public Object getSourceId() {

        return this.getCSVParameters().getFile();

    }

    public String getName() {

        String name = this.getCSVParameters().getFile().getName();

        return FilenameUtils.getBaseName(name);

    }

    public String getFullName() {

        return this.getCSVParameters().getFile().getAbsolutePath();

    }

    public ResourceProvider getResource() {

        return resource;

    }

    private boolean isEmpty(String s) {

        if (s == null) {

            return true;

        }

        return s.trim().length() == 0;

    }

    private class FieldTypeParser {

        public String name = null;

        public int type = DataTypes.STRING;

        public int size = 0;

        public boolean allowNulls = true;

        public int geometryType = Geometry.TYPES.GEOMETRY;

        private String typename = "string";

        FieldTypeParser() {

        }

        private int getType(String value) {

            DataTypesManager dataTypesManager = ToolsLocator.getDataTypesManager();

            return dataTypesManager.getType(typename);

        }

        public void clear() {

            name = null;

            type = DataTypes.STRING;

            size = 0;

            allowNulls = true;

            geometryType = Geometry.TYPES.GEOMETRY;

        }

        public void copyFrom(FieldTypeParser other) {

            name = other.name;

            type = other.type;

            size = other.size;

            allowNulls = other.allowNulls;

            geometryType = other.geometryType;

        }

        // El formato seria:

        //   name[:typename[:size[:notnull|null]]]

        //   name[:GEOMETRY[:geometry_type[:notnull|null]]]

        //   name[__typename[__size[__notnull|null]]]

        //

        public boolean parse(String value) {

            String typename = null;

            String[] ss = null;

            if (value.contains(":")) {

                ss = value.split(":");

            } else if (value.contains("__")) {

                ss = value.split("__");

            }

            if (ss == null) {

                this.name = value;

                return true;

            }

            switch (ss.length) {

                case 4:

                    if (!StringUtils.isBlank(ss[3])) {

                        if ("notnull".equalsIgnoreCase(ss[3].trim())) {

                            this.allowNulls = false;

                        } else {

                            this.allowNulls = true;

                        }

                    }

                case 3:

                    if (!StringUtils.isBlank(ss[1])) {

                        this.typename = ss[1].trim();

                        this.type = this.getType(this.typename);

                        if (this.type == DataTypes.INVALID) {

                            this.type = DataTypes.STRING;

                            logger.info("Type '" + typename + "' not valid for attribute '" + value + "' in CSV file '" + getFullFileName() + "'.");

                        }

                    }

                    if (!StringUtils.isBlank(ss[2])) {

                        if (this.type == DataTypes.GEOMETRY) {

                            String s = ss[2].trim();

                            if (s.equalsIgnoreCase("line") || s.equalsIgnoreCase("linestring") || s.equalsIgnoreCase("curve")) {

                                this.geometryType = Geometry.TYPES.LINE;

                            } else if (s.equalsIgnoreCase("multiline") || s.equalsIgnoreCase("multilinestring") || s.equalsIgnoreCase("multicurve")) {

                                this.geometryType = Geometry.TYPES.MULTILINE;

                            } else if (s.equalsIgnoreCase("point")) {

                                this.geometryType = Geometry.TYPES.POINT;

                            } else if (s.equalsIgnoreCase("multipoint")) {

                                this.geometryType = Geometry.TYPES.MULTIPOINT;

                            } else if (s.equalsIgnoreCase("polygon") || s.equalsIgnoreCase("surface")) {

                                this.geometryType = Geometry.TYPES.POLYGON;

                            } else if (s.equalsIgnoreCase("multipolygon") || s.equalsIgnoreCase("multisurface")) {

                                this.geometryType = Geometry.TYPES.MULTIPOLYGON;

                            }

                            this.size = 1;

                        } else {

                            try {

                                this.size = Integer.parseInt(ss[2]);

                            } catch (Exception ex) {

                                logger.warn("Ignore incorrect field size for field " + value + " in CSV header of '" + getFullFileName() + "'.", ex);

                            }

                        }

                    }

                    this.name = ss[0].trim();

                    break;

                case 2:

                    if (!StringUtils.isBlank(ss[1])) {

                        this.typename = ss[1].trim();

                        this.type = this.getType(this.typename);

                        if (this.type == DataTypes.INVALID) {

                            this.type = DataTypes.STRING;

                            logger.info("Type '" + typename + "' not valid for attribute '" + value + "' in CSV file '" + getFullFileName() + "'.");

                        }

                    }

                case 1:

                    this.name = ss[0].trim();

                    break;

            }

            if (this.type != DataTypes.STRING) {

                this.size = 0;

            }

            return true;

        }

    }

    private EditableFeatureType getFeatureType(String headers[], int automaticTypes[]) {

        EditableFeatureType fType = getStoreServices().createFeatureType(this.getName());

        fType.setHasOID(true);

        FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length];

        //

        // Calculamos cuales pueden ser los tipos de datos

        //

        for (int i = 0; i < fieldTypes.length; i++) {

            fieldTypes[i] = new FieldTypeParser();

        }

        // Asuminos los tipos pasados por parametro, que se supone

        // son los detectados automaticamente.

        if (automaticTypes != null) {

            for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) {

                fieldTypes[i].type = automaticTypes[i];

            }

        }

        // Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo

        // los tipos anteriores en caso de definirse en la cabezara.

        for (int i = 0; i < fieldTypes.length; i++) {

            if (!fieldTypes[i].parse(headers[i])) {

                continue;

            }

        }

        // Y por ultimo hacemos caso a lo que se haya especificado en los parametros

        // de apertura del CSV, teniendo esto prioridad sobre todo.

        String param_types_def = CSVStoreParameters.getRawFieldTypes(this.getParameters());

        if (StringUtils.isNotBlank(param_types_def)) {

            String sep = CSVStoreParameters.getDelimiter(param_types_def);

            if (StringUtils.isNotBlank(sep)) {

                String[] param_types = param_types_def.split(sep);

                FieldTypeParser parser = new FieldTypeParser();

                for (String param_type : param_types) {

                    parser.clear();

                    parser.parse(param_type);

                    for (FieldTypeParser fieldType : fieldTypes) {

                        if (StringUtils.equalsIgnoreCase(fieldType.name, parser.name)) {

                            fieldType.copyFrom(parser);

                            break;

                        }

                    }

                }

            }

        }

        //

        // Una vez ya sabemos los tipos de datos rellenamos el feature-type

        //

        for (int i = 0; i < fieldTypes.length; i++) {

            EditableFeatureAttributeDescriptor fad = fType.add(

                    fieldTypes[i].name,

                    fieldTypes[i].type

            );

            fad.setSize(fieldTypes[i].size);

            fad.setAllowNull(fieldTypes[i].allowNulls);

            if (fieldTypes[i].type == DataTypes.GEOMETRY) {

                if (fType.getDefaultGeometryAttributeName() == null) {

                    fType.setDefaultGeometryAttributeName(fieldTypes[i].name);

                }

                fad.setGeometryType(fieldTypes[i].geometryType);

            }

        }

        String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters());

        if (pointDimensionNames != null) {

            PointAttributeEmulator emulator = new PointAttributeEmulator(pointDimensionNames);

            EditableFeatureAttributeDescriptor attr = fType.add(

                    CSVStoreParameters.getPointColumnName(this.getParameters()), 

                    DataTypes.GEOMETRY, emulator

            );

            GeometryManager geommgr = GeometryLocator.getGeometryManager();

            GeometryType gt;

            try {

                gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM3D);

                attr.setGeometryType(gt);

                attr.setGeometryType(Geometry.TYPES.POINT);

                attr.setGeometrySubType(Geometry.SUBTYPES.GEOM3D);

            } catch (Exception e) {

                logger.warn("Can't set geometry type for the calculated field in CSV file '" + getFullFileName() + "'.", e);

            }

        }

        String geometry_column = CSVStoreParameters.getGeometryColumn(this.getParameters());

        if( !StringUtils.isEmpty(geometry_column) ) {

            EditableFeatureAttributeDescriptor attr = (EditableFeatureAttributeDescriptor) fType.get(geometry_column);

            if( attr!=null && attr.getType()!=DataTypes.GEOMETRY ) {

                attr.setDataType(DataTypes.GEOMETRY);

                GeometryManager geommgr = GeometryLocator.getGeometryManager();

                GeometryType gt;

                try {

                    gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.UNKNOWN);

                    attr.setGeometryType(gt);

                } catch (Exception e) {

                    logger.warn("Can't set geometry type for the calculated field in CSV file '" + getFullFileName() + "'.", e);

                }

                fType.setDefaultGeometryAttributeName(geometry_column);

            }

        }

        return fType;

    }

    static class PointAttributeEmulator implements FeatureAttributeEmulator {

        private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class);

        private static final int XNAME = 0;

        private static final int YNAME = 1;

        private static final int ZNAME = 2;

        private final GeometryManager geommgr;

        private final String[] fieldNames;

        private final Coercion toDouble;

        private final DataType dataType;

        private int errorcount = 0;

        public PointAttributeEmulator(String[] pointDimensionNames) {

            if (pointDimensionNames.length > 2) {

                this.fieldNames = new String[3];

                this.fieldNames[ZNAME] = pointDimensionNames[2];

            } else {

                this.fieldNames = new String[2];

            }

            this.fieldNames[XNAME] = pointDimensionNames[0];

            this.fieldNames[YNAME] = pointDimensionNames[1];

            this.geommgr = GeometryLocator.getGeometryManager();

            DataTypesManager datatypeManager = ToolsLocator.getDataTypesManager();

            this.toDouble = datatypeManager.getCoercion(DataTypes.DOUBLE);

            this.dataType = datatypeManager.get(DataTypes.GEOMETRY);

        }

        public Object get(Feature feature) {

            try {

                Object valueX = feature.get(this.fieldNames[XNAME]);

                valueX = toDouble.coerce(valueX);

                if (valueX == null) {

                    return null;

                }

                Object valueY = feature.get(this.fieldNames[YNAME]);

                valueY = toDouble.coerce(valueY);

                if (valueY == null) {

                    return null;

                }

                Object valueZ = null;

                if (this.fieldNames.length > 2) {

                    valueZ = toDouble.coerce(feature.get(this.fieldNames[ZNAME]));

                    if (valueZ == null) {

                        return null;

                    }

                }

                double x = ((Double) valueX).doubleValue();

                double y = ((Double) valueY).doubleValue();

                Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);

                if (this.fieldNames.length > 2) {

                    double z = ((Double) valueZ).doubleValue();

                    point.setCoordinateAt(2, z);

                }

                return point;

            } catch (Exception ex) {

                if (++errorcount < 5) {

                    logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='"

                            + this.fieldNames[XNAME] + "', YNAME='" + this.fieldNames[XNAME] + "' feature=" + feature.toString(), ex);

                }

                return null;

            }

        }

        public void set(EditableFeature feature, Object value) {

            if (value == null) {

                return;

            }

            Point point = null;

            if (value instanceof MultiPoint) {

                point = (Point) ((MultiPoint) value).getPrimitiveAt(0);

            } else {

                point = (Point) value;

            }

            feature.set(this.fieldNames[XNAME], point.getX());

            feature.set(this.fieldNames[YNAME], point.getY());

            if (this.fieldNames.length > 2) {

                feature.set(this.fieldNames[ZNAME], point.getCoordinateAt(2));

            }

        }

        public boolean allowSetting() {

            return true;

        }

        public String[] getRequiredFieldNames() {

            return this.fieldNames;

        }

    }

    static class ToPointEvaluaror extends AbstractEvaluator {

        private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class);

        private GeometryManager geommgr = null;

        private String xname = null;

        private String yname = null;

        private String zname = null;

        private Coercion toDouble;

        private int errorcount = 0;

        ToPointEvaluaror(String[] pointDimensionNames) {

            this.xname = pointDimensionNames[0];

            this.yname = pointDimensionNames[1];

            if (pointDimensionNames.length > 2) {

                this.zname = pointDimensionNames[2];

            }

            this.geommgr = GeometryLocator.getGeometryManager();

            this.toDouble = ToolsLocator.getDataTypesManager().getCoercion(DataTypes.DOUBLE);

        }

        public Object evaluate(EvaluatorData data) throws EvaluatorException {

            try {

                double x = ((Double) toDouble.coerce(data.getDataValue(xname))).doubleValue();

                double y = ((Double) toDouble.coerce(data.getDataValue(yname))).doubleValue();

                Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);

                if (zname != null) {

                    double z = ((Double) toDouble.coerce(data.getDataValue(zname))).doubleValue();

                    point.setCoordinateAt(2, z);

                }

                return point;

            } catch (Exception ex) {

                if (++errorcount < 5) {

                    logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='"

                            + xname + "', YNAME='" + yname + "', ZNAME='" + zname + "', data=" + data.toString());

                }

                return null;

            }

        }

        public String getName() {

            return "ToPointEvaluaror";

        }

    }

    private SimpleReader getSimpleReader(InputStreamReader in) {

        SimpleReader reader;

        if (CSVStoreParameters.getRawFieldsDefinition(getCSVParameters()) != null) {

            reader = new FixedLenReader(in, getCSVParameters());

        } else {

            reader = new CSVReader(in, getCSVParameters());

        }

        return reader;

    }

    private String getFixedHeader(int column) {

        char[] header = new char[3];

        String s = String.format("%03d", column);

        header[0] = (char) (s.charAt(0) + 17);

        header[1] = (char) (s.charAt(1) + 17);

        header[2] = (char) (s.charAt(2) + 17);

        return String.valueOf(header);

    }

    private String[] getFixedHeaders(int count) {

        String[] headers = new String[count];

        for (int i = 0; i < headers.length; i++) {

            headers[i] = getFixedHeader(i);

        }

        return headers;

    }

    private InputStreamReader openFile(File f, String charsetName) throws FileNotFoundException {

        Charset charset = Charset.defaultCharset();

        FileInputStream fis = new FileInputStream(f);

        if( !StringUtils.isEmpty(charsetName) ) {

            if( Charset.isSupported(charsetName) )  {

                try {

                    charset = Charset.forName(charsetName);

                } catch(Throwable th) {

                    logger.warn("Can't use charset '"+charsetName+"' for read csv '"+this.getFullFileName()+"'.", th);

                }

            } else {

                logger.warn("charset '"+charsetName+"' not supported for read csv '"+this.getFullFileName()+"'.");                

            }

        }

        InputStreamReader isr = new InputStreamReader(fis, charset);

        return isr;

    }

    private void loadFeatures() {

        InputStreamReader in = null;

        SimpleReader reader = null;

        try {

            String headers[] = null;

            FeatureStoreProviderServices store = this.getStoreServices();

            boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());

            in = openFile(

                this.getCSVParameters().getFile(), 

                CSVStoreParameters.getCharset(this.getCSVParameters())

            );

            reader = getSimpleReader(in);

            headers = CSVStoreParameters.getHeaders(getCSVParameters());

            if (headers == null) {

                if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {

                    headers = reader.getHeader();

                    if (headers == null) {

                        if (CSVStoreParameters.getIgnoreErrors(getCSVParameters())) {

                            headers = getFixedHeaders(reader.getColumnsCount());

                        } else {

                            String msg = "Can't retrieve header from csv file '"

                                    + this.getCSVParameters().getFile()

                                    .getAbsolutePath()

                                    + "' and not specified in the parameters.";

                            logger.warn(msg);

                            throw new RuntimeException(msg);

                        }

                    }

                } else {

                    headers = getFixedHeaders(reader.getColumnsCount());

                }

            } else {

                if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {

                    reader.getHeader(); // Skip and ignore the header of file

                }

            }

            int[] detectedTypes = automaticDetectionOfTypes(headers);

            if( detectedTypes!=null && detectedTypes.length>headers.length ) {

                // Se han detectado mas columnas que las que hay en la cabezera,

                // a?adimos mas columnas a la cabezera.

                String[] headers2 = new String[detectedTypes.length];

                for( int i=0; i<headers2.length; i++ ) {

                    if( i<headers.length ) {

                        headers2[i] = headers[i];

                    } else {

                        headers2[i] = getFixedHeader(i);

                    }

                }

                headers = headers2;

            }

            // Initialize the feature types

            EditableFeatureType edftype = this.getFeatureType(headers, detectedTypes);

            FeatureType ftype = edftype.getNotEditableCopy();

            List<FeatureType> ftypes = new ArrayList<>();

            ftypes.add(ftype);

            store.setFeatureTypes(ftypes, ftype);

            Coercion coercion[] = new Coercion[ftype.size()];

            int sizes[] = new int[ftype.size()];

            for (int i = 0; i < ftype.size(); i++) {

                sizes[i] = -1;

                FeatureAttributeDescriptor ad = ftype.getAttributeDescriptor(i);

                coercion[i] = ad.getDataType().getCoercion();

                switch (ad.getDataType().getType() ) {

                case DataTypes.STRING:

                    if (ad.getSize() == 0) {

                        // Es un string y no tiene un size asignado.

                        // Lo ponemos a cero para calcularlo.

                        sizes[i] = 0;

                    }

                    break;

                case DataTypes.URL:

                case DataTypes.URI:

                case DataTypes.FILE:

                    sizes[i] = 0;

                }

            }

            if (ftype.getDefaultGeometryAttributeName() != null) {

                this.need_calculate_envelope = true;

            }

            Locale locale = CSVStoreParameters.getLocale(getCSVParameters());

            taskStatus.message("_loading");

            int count = 0;

            int count_errors = 0;

            List<String> row = reader.read();

            int skipLines = CSVStoreParameters.getSkipLines(getCSVParameters());

            if (skipLines > 0) {

                row = reader.skip(skipLines);

            }

            int limit = CSVStoreParameters.getLimit(getCSVParameters());

            while (row != null) {

                taskStatus.setCurValue(++count);

                FeatureProvider feature = this.createFeatureProvider(ftype);

                for (int i = 0; i < row.size(); i++) {

                    Object rawvalue = row.get(i);

                    try {

                        Object value = null;

                        if (locale != null && coercion[i] instanceof CoercionWithLocale) {

                            value = ((CoercionWithLocale) (coercion[i])).coerce(rawvalue, locale);

                        } else {

                            value = coercion[i].coerce(rawvalue);

                        }

                        feature.set(i, value);

                        if (sizes[i] >= 0 && 

                            (value instanceof String || value instanceof URL || 

                             value instanceof URI || value instanceof File) ) {

                            int x = value.toString().length();

                            if (sizes[i] < x) {

                                sizes[i] = x;

                            }

                        }

                    } catch (Exception ex) {

                        if (!ignore_errors) {

                            throw ex;

                        }

                        if (count_errors++ < 10) {

                            logger.warn("Can't load value of attribute " + i + " in row " + count + ".", ex);

                        }

                        if (count_errors == 10) {

                            logger.info("Too many errors, suppress messages.");

                        }

                    }

                }

                this.addFeatureProvider(feature);

                if( limit>0 ) {

                    if( limit < this.data.size() ) {

                        break;

                    }

                }

                row = reader.read();

            }

            for (int i = 0; i < ftype.size(); i++) {

                if (sizes[i] > 0) {

                    EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor) edftype.getAttributeDescriptor(i));

                    efad.setSize(sizes[i]);

                }

            }

            // Volvemos a asignar al store el featuretype, ya que puede

            // haber cambiado.

            ftype = edftype.getNotEditableCopy();

            ftypes = new ArrayList<>();

            ftypes.add(ftype);

            store.setFeatureTypes(ftypes, ftype);

            taskStatus.terminate();

        } catch (Exception ex) {

            int lineno = 0;

            if (reader != null) {

                lineno = reader.getLine();

            }

            throw new RuntimeException("Problems reading file '" + getFullFileName() + "' near line " + lineno + ".", ex);

        } finally {

            if (reader != null) {

                try {

                    reader.close();

                } catch (Exception ex) {

                    // Do nothing

                }

                reader = null;

            }

            if (in != null) {

                try {

                    in.close();

                } catch (Exception ex) {

                    // Do nothing

                }

                in = null;

            }

        }

    }

    private static class PossibleDataType {

        public boolean possibleInt = true;

        public boolean possibleFloat = true;

        public boolean possibleDouble = true;

        public boolean possibleLong = true;

        public boolean possibleURL = true;

        public boolean possibleDate = true;

        public boolean possibleGeometry = true;

    }

    private int[] automaticDetectionOfTypes(String[] headers) throws IOException {

        boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(getCSVParameters());

        if (!automatic_types_detection) {

            return null;

        }

        List<PossibleDataType> possibleDataTypes;

        Locale locale;

        int[] types = null;

        FileReader in = null;

        SimpleReader reader = null;

        try {

            in = new FileReader(this.getCSVParameters().getFile());

            reader = getSimpleReader(in);

            if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {

                reader.read();

            }

            possibleDataTypes = new ArrayList<>(headers.length);

            for (String header : headers) {

                possibleDataTypes.add(new PossibleDataType());

            }

            locale = CSVStoreParameters.getLocale(getCSVParameters());

            if (locale == null) {

                locale = Locale.getDefault();

            }

            DataTypesManager typeManager = ToolsLocator.getDataTypesManager();

            CoercionWithLocale toDouble = (CoercionWithLocale) typeManager.getCoercion(DataTypes.DOUBLE);

            CoercionWithLocale toFloat = (CoercionWithLocale) typeManager.getCoercion(DataTypes.FLOAT);

            CoercionWithLocale toDate = (CoercionWithLocale) typeManager.getCoercion(DataTypes.DATE);

            CoercionWithLocale toInt = (CoercionWithLocale) typeManager.getCoercion(DataTypes.INT);

            CoercionWithLocale toLong = (CoercionWithLocale) typeManager.getCoercion(DataTypes.LONG);

            Coercion toGeom = typeManager.getCoercion(DataTypes.GEOMETRY);

            List<String> row = reader.read();

            while (row != null) {

                for (int i = 0; i < row.size(); i++) {

                    while( possibleDataTypes.size()<row.size() ) {

                        possibleDataTypes.add(new PossibleDataType());

                    }

                    String rawvalue = row.get(i);

                    PossibleDataType possibleDataType = possibleDataTypes.get(i);

                    if (possibleDataType.possibleDouble) {

                        try {

                            toDouble.coerce(rawvalue, locale);

                            possibleDataType.possibleDouble = true;

                        } catch (Exception ex) {

                            possibleDataType.possibleDouble = false;

                        }

                    }

                    if (possibleDataType.possibleFloat) {

                        try {

                            toFloat.coerce(rawvalue, locale);

                            possibleDataType.possibleFloat = true;

                        } catch (Exception ex) {

                            possibleDataType.possibleFloat = false;

                        }

                    }

                    if (possibleDataType.possibleLong) {

                        possibleDataType.possibleLong = isValidLong(rawvalue);

                    }

                    if (possibleDataType.possibleInt) {

                        possibleDataType.possibleInt = isValidInteger(rawvalue);

                    }

                    if (possibleDataType.possibleDate) {

                        try {

                            toDate.coerce(rawvalue, locale);

                            possibleDataType.possibleDate = true;

                        } catch (Exception ex) {

                            possibleDataType.possibleDate = false;

                        }

                    }

                    if (possibleDataType.possibleURL) {

                        try {

                            new URL((String) rawvalue);

                            possibleDataType.possibleURL = true;

                        } catch (Exception ex) {

                            possibleDataType.possibleURL = false;

                        }

                    }

                    if (possibleDataType.possibleGeometry) {

                        try {

                            toGeom.coerce((String) rawvalue);

                            possibleDataType.possibleGeometry = true;

                        } catch (Exception ex) {

                            possibleDataType.possibleGeometry = false;

                        }