Application: gvSIG desktop

org.gvsig.fmap.dal.store.csv.CSVLibrary

/**

 * gvSIG. Desktop Geographic Information System.

 *

 * Copyright (C) 2007-2013 gvSIG Association.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 3

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

 * MA  02110-1301, USA.

 *

 * For any additional information, do not hesitate to contact us

 * at info AT gvsig.com, or visit our website www.gvsig.com.

 */

package org.gvsig.fmap.dal.store.csv;

import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.FeatureType;

import org.gvsig.fmap.dal.feature.spi.DefaultFeatureProvider;

import org.gvsig.fmap.dal.feature.spi.FeatureProvider;

import org.gvsig.fmap.geom.Geometry;

import org.gvsig.fmap.geom.primitive.Envelope;

public abstract class AbstractFeatureProviderLoadedOnDemand extends DefaultFeatureProvider {

    protected boolean loading;

    protected boolean loaded;

    protected String geomName;

    public AbstractFeatureProviderLoadedOnDemand(FeatureType type) {

        super(type);

        loading = false;

        loaded = false;

        FeatureAttributeDescriptor geomdesc = type.getDefaultGeometryAttribute();

        if( geomdesc!=null ) {

            geomName = geomdesc.getName();

        }

    }

    protected void load() {

        if (loading || loaded || this.isNew()) {

            return;

        }

        loading = true;

        try {

            doLoad();

        } catch (Exception e) {

        } finally {

            loading = false;

            loaded = true;

        }

    }

    protected abstract void doLoad();

    @Override

    public void set(int i, Object value) {

        this.load();

        super.set(i, value);

    }

    @Override

    public void set(String name, Object value) {

        this.load();

        super.set(featureType.getIndex(name), value);

    }

    @Override

    public Object get(int i) {

        this.load();

        return super.get(i);

    }

    @Override

    public Object get(String name) {

        this.load();

        return super.get(name);

    }

    @Override

    public Geometry getDefaultGeometry() {

        if( geomName == null ) {

            return null;

        }

        return (Geometry) this.get(geomName);

    }

    @Override

    public Envelope getDefaultEnvelope() {

        Geometry geom = this.getDefaultGeometry();

        if( geom==null ) {

            return null;

        }

        try {

            return geom.getEnvelope();

        } catch(Exception ex) {

            // En ocasiones con geometrias incorrectas getEnvelop falla.

            return null;

        }

    }

    @Override

    public void setOID(Object oid) {

        this.loaded = false;

        super.setOID(oid);

    }

    @Override

    public FeatureProvider getCopy() {

        this.load();

        return super.getCopy();

    }

}

/**

 * gvSIG. Desktop Geographic Information System.

 *

 * Copyright (C) 2007-2013 gvSIG Association.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 3

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

 * MA 02110-1301, USA.

 *

 * For any additional information, do not hesitate to contact us

 * at info AT gvsig.com, or visit our website www.gvsig.com.

 */

package org.gvsig.fmap.dal.store.csv;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.Reader;

import java.nio.charset.Charset;

import java.util.Locale;

import java.util.Map;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.io.input.BOMInputStream;

import org.apache.commons.lang3.ArrayUtils;

import org.apache.commons.lang3.StringUtils;

import org.gvsig.fmap.dal.DataTypes;

import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.EditableFeatureType;

import static org.gvsig.fmap.dal.store.csv.CSVStoreProvider.NAME;

import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReaderSuperCSV;

import org.gvsig.fmap.dal.store.csv.simplereaders.FixedLenReader;

import org.gvsig.fmap.dal.store.csv.simplereaders.GMLReader;

import org.gvsig.fmap.dal.store.csv.simplereaders.JSonReader;

import org.gvsig.fmap.dal.store.simplereader.AutomaticDetectionOfTypes;

import org.gvsig.fmap.dal.store.simplereader.FieldTypeParser;

import org.gvsig.fmap.dal.store.simplereader.SimpleReader;

import org.gvsig.fmap.geom.Geometry;

import org.gvsig.fmap.geom.GeometryLocator;

import org.gvsig.fmap.geom.GeometryManager;

import org.gvsig.fmap.geom.type.GeometryType;

import org.gvsig.tools.dynobject.Tags;

import org.gvsig.tools.task.SimpleTaskStatus;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

/**

 *

 * @author gvSIG  Team

 */

@SuppressWarnings("UseSpecificCatch")

public class CSVUtils {

    private static final Logger LOGGER = LoggerFactory.getLogger(CSVUtils.class);

    private CSVUtils() {

    }

    public static InputStreamReader openFile(File f, String charsetName) throws FileNotFoundException {

        String fullFileName = f==null? "NULL":f.getAbsolutePath();

        Charset charset = Charset.defaultCharset();

        InputStream fis = new BOMInputStream(new FileInputStream(f));

        if (StringUtils.isNotBlank(charsetName)) {

            if (Charset.isSupported(charsetName)) {

                try {

                    charset = Charset.forName(charsetName);

                } catch (Throwable th) {

                    LOGGER.warn("Can't use charset '" + charsetName + "' for read csv '" + fullFileName + "'.", th);

                }

            } else {

                LOGGER.warn("charset '" + charsetName + "' not supported for read csv '" + fullFileName + "'.");

            }

        }

        InputStreamReader isr = new InputStreamReader(fis, charset);

        return isr;

    }

    public static boolean loadFeatureType(CSVStoreParameters parameters, EditableFeatureType featureType, boolean  detectTypes, SimpleTaskStatus status) throws IOException {

        InputStreamReader in = null;

        SimpleReader reader = null;

        try {

            String headers[];

            in = openFile(

                    parameters.getFile(),

                    CSVStoreParameters.getCharset(parameters)

            );

            reader = getSimpleReader(parameters, in);

            headers = CSVStoreParameters.getHeaders(parameters);

            if (headers == null) {

                if (CSVStoreParameters.isFirstLineHeader(parameters)) {

                    headers = reader.getHeader();

                    if (headers == null) {

                        if (CSVStoreParameters.getIgnoreErrors(parameters)) {

                            headers = getFixedHeaders(reader.getColumnsCount());

                        } else {

                            String msg = "Can't retrieve header from csv file '"

                                    + parameters.getFile()

                                            .getAbsolutePath()

                                    + "' and not specified in the parameters.";

                            LOGGER.warn(msg);

                            throw new RuntimeException(msg);

                        }

                    }

                } else {

                    headers = getFixedHeaders(reader.getColumnsCount());

                }

            } else {

                if (CSVStoreParameters.isFirstLineHeader(parameters)) {

                    reader.getHeader(); // Skip and ignore the header of file

                }

            }

            AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = null;

            if( detectTypes ) {

                detectedTypes = automaticDetectionOfTypes(parameters, headers, status);

            }

            if( StringUtils.isBlank(headers[headers.length-1]) &&

                (detectedTypes==null || detectedTypes[headers.length-1].isBlank()) ) {

                headers = ArrayUtils.remove(headers, headers.length-1);

            }

            if (detectedTypes != null && detectedTypes.length > headers.length) {

                // Se han detectado mas columnas que las que hay en la cabezera,

                // a?adimos mas columnas a la cabezera.

                String[] headers2 = new String[detectedTypes.length];

                for (int i = 0; i < headers2.length; i++) {

                    if (i < headers.length) {

                        headers2[i] = headers[i];

                    } else {

                        headers2[i] = getFixedHeader(i);

                    }

                }

                headers = headers2;

            }

            for (int i = 0; i < headers.length; i++) {

                if (StringUtils.isBlank(headers[i])) {

                    headers[i] = getFixedHeader(i);

                }

            }

            // Initialize the feature types

            return fillFeatureType(parameters, featureType,  headers, detectedTypes);

        } finally {

            IOUtils.closeQuietly(in);

            IOUtils.closeQuietly(reader);

        }

    }

    public static SimpleReader getSimpleReader(CSVStoreParameters parameters, Reader in) throws IOException {

        SimpleReader reader;

        String filename = CSVStoreParameters.getFileName(parameters);

        if (FilenameUtils.isExtension(filename, "json")){

            reader= new JSonReader(in,parameters);

        } else if (FilenameUtils.isExtension(filename, "gml")){

            reader= new GMLReader(in,parameters);

        } else if (CSVStoreParameters.getRawFieldsDefinition(parameters) != null) {

            reader = new FixedLenReader(in, parameters);

        } else {

            reader = new CSVReaderSuperCSV(in, parameters);

        }

        return reader;

    }

    private static String getFixedHeader(int column) {

        char[] header = new char[3];

        String s = String.format("%03d", column);

        header[0] = (char) (s.charAt(0) + 17);

        header[1] = (char) (s.charAt(1) + 17);

        header[2] = (char) (s.charAt(2) + 17);

        return String.valueOf(header);

    }

    private static String[] getFixedHeaders(int count) {

        String[] headers = new String[count];

        for (int i = 0; i < headers.length; i++) {

            headers[i] = getFixedHeader(i);

        }

        return headers;

    }

    private static AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(CSVStoreParameters parameters, String[] headers, SimpleTaskStatus status) throws IOException {

        String fullFileName = parameters.getFile()==null? "NULL":parameters.getFile().getAbsolutePath();

        boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(parameters);

        if (!automatic_types_detection) {

            return null;

        }

        AutomaticDetectionOfTypes.DetectedValue[] types = null;

        Reader in = null;

        SimpleReader reader = null;

        try {

            in = openFile(

                    parameters.getFile(),

                    CSVStoreParameters.getCharset(parameters)

            );

            reader = getSimpleReader(parameters, in);

            AutomaticDetectionOfTypes x = new AutomaticDetectionOfTypes(

                    fullFileName

            );

            types = x.detect(

                    headers.length,

                    reader,

                    CSVStoreParameters.isFirstLineHeader(parameters),

                    CSVStoreParameters.getLocale(parameters),

                    status

            );

        } catch (Exception ex) {

            int lineno = 0;

            if (reader != null) {

                lineno = reader.getLine();

            }

            throw new RuntimeException("Problems reading file '" + fullFileName + "' near line " + lineno + ".", ex);

        } finally {

            IOUtils.closeQuietly(reader);

            IOUtils.closeQuietly(in);

        }

        return types;

    }

    private static boolean fillFeatureType(CSVStoreParameters parameters, EditableFeatureType fType, String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) {

        String fullFileName = parameters.getFile()==null? "":parameters.getFile().getAbsolutePath();

        String providerName = NAME;

        fType.setHasOID(true);

        FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length];

        //

        // Calculamos cuales pueden ser los tipos de datos

        //

        for (int i = 0; i < fieldTypes.length; i++) {

            fieldTypes[i] = new FieldTypeParser(providerName, fullFileName);

        }

        // Asuminos los tipos pasados por parametro, que se supone

        // son los detectados automaticamente.

        if (automaticTypes != null) {

            for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) {

                fieldTypes[i].detectedValue = automaticTypes[i];

                fieldTypes[i].type = automaticTypes[i].getType();

            }

        }

        // Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo

        // los tipos anteriores en caso de definirse en la cabezara.

        boolean all_fields_declare_type = true;

        for (int i = 0; i < fieldTypes.length; i++) {

            if (!fieldTypes[i].parse(headers[i])) {

                LOGGER.warn("Can't parse header of field "+i+ "( "+headers[i]+") in '"+providerName+"' file '" + fullFileName + "'.");

            }

            if( fieldTypes[i].type == DataTypes.UNKNOWN ) {

                all_fields_declare_type = false;

                fieldTypes[i].type = DataTypes.STRING;

            }

        }

        // Y por ultimo hacemos caso a lo que se haya especificado en los parametros

        // de apertura del CSV, teniendo esto prioridad sobre todo.

        String param_types_def = CSVStoreParameters.getRawFieldTypes(parameters);

        if (StringUtils.isNotBlank(param_types_def)) {

            String sep = CSVStoreParameters.getDelimiter(param_types_def);

            if (StringUtils.isNotBlank(sep)) {

                String[] param_types = param_types_def.split(sep);

                FieldTypeParser parser = new FieldTypeParser(providerName, fullFileName);

                for (String param_type : param_types) {

                    parser.clear();

                    parser.parse(param_type);

                    for (FieldTypeParser fieldType : fieldTypes) {

                        if (StringUtils.equalsIgnoreCase(fieldType.name, parser.name)) {

                            fieldType.copyFrom(parser);

                            break;

                        }

                    }

                }

            }

        }

        //

        // Una vez ya sabemos los tipos de datos rellenamos el feature-type

        //

        Tags ftypeTags = fType.getTags();

        for (FieldTypeParser fieldType : fieldTypes) {

            EditableFeatureAttributeDescriptor fad = fType.add(fieldType.name, fieldType.type);

            if( fieldType.detectedValue!=null ) {

                fad.setDisplaySize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));

                fad.setSize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));

                if( fad.getPrecision()<fieldType.detectedValue.getPrecision() ) {

                    fad.setPrecision(fieldType.detectedValue.getPrecision());

                }

                if( fad.getScale()<fieldType.detectedValue.getScale()) {

                    fad.setScale(fieldType.detectedValue.getScale());

                }

            } else {

                fad.setDisplaySize(fieldType.size);

            }

            if (fieldType.type == DataTypes.GEOMETRY ) {

                fad.setGeometryType(fieldType.geomType, fieldType.geomSubtype);

                if( fType.getDefaultGeometryAttributeName() == null ) {

                    fType.setDefaultGeometryAttributeName(fieldType.name);

                }

            } 

            Locale locale = null;

            if (fieldType.type == DataTypes.TIMESTAMP ) {

                if(!CSVStoreParameters.isBlankOrDefaultLocale(parameters)){

                    locale = CSVStoreParameters.getLocale(parameters);

                }

            } else {

                locale = CSVStoreParameters.getLocale(parameters);

            }

            fad.setLocale(locale);

            for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) {

                try {

                    switch(entry.getKey().toLowerCase()) {

                        case "expression":

                            // Los campos calculados los procesamos en una segunda

                            // pasada, cuando ya estan definidos el resto de los campos

                            // ya que pueden requerir campos que aun no se han definido.

                            break;

                        default:

                                fad.set(entry.getKey(), entry.getValue());

                            }

                } catch (Exception ex) {

                    LOGGER.warn("Can't set property '"+entry.getKey()+"' of '"+fad.getName()+"'.", ex);

                }

            }            

            Tags tags = fad.getTags();

            for (Map.Entry<String, String> entry : fieldType.tags.entrySet()) {

                tags.set(entry.getKey(), entry.getValue());

            }

            for (Map.Entry<String, String> entry : fieldType.typetags.entrySet()) {

                ftypeTags.set(entry.getKey(), entry.getValue());

            }

            for (Map.Entry<String, String> entry : fieldType.typeAssignments.entrySet()) {

                try {

                    fType.set(entry.getKey(), entry.getValue());

                } catch(Exception ex) {

                    LOGGER.warn("Can't set attribute '"+entry.getKey()+"' in the feature type.", ex);

                }

            }

        }

        // Processamos ahora los campos calculados

        for (FieldTypeParser fieldType : fieldTypes) {

            EditableFeatureAttributeDescriptor fad = fType.getEditableAttributeDescriptor(fieldType.name);

            for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) {

                try {

                    switch(entry.getKey().toLowerCase()) {

                        case "expression":

                            fad.set(entry.getKey(), entry.getValue());

                            break;

                    }

                } catch (Exception ex) {

                    LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+fullFileName+"'.", ex);

                }

            }

        }

        String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(parameters);

        if ( pointDimensionNames != null ) {

            CSVPointAttributeEmulator emulator = new CSVPointAttributeEmulator(pointDimensionNames);

            String columnName = CSVStoreParameters.getPointColumnName(parameters);

            if( StringUtils.isBlank(columnName) ) {

                columnName = "geom";

            }

            EditableFeatureAttributeDescriptor attr = fType.add(columnName, DataTypes.GEOMETRY, emulator);

            GeometryManager geommgr = GeometryLocator.getGeometryManager();

            GeometryType gt;

            try {

                if ( emulator.getFieldNames() != null && emulator.getFieldNames().length <= 2 ) {

                	gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM2D);

                } else {

                	gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM3D);

                }

                attr.setGeometryType(gt);

            } catch (Exception e) {

                LOGGER.warn("Can't set geometry type for the calculated field in '"+providerName+"' file '" + fullFileName + "'.", e);

            }

        }        

        String geometry_column = CSVStoreParameters.getGeometryColumn(parameters);

        if (!StringUtils.isEmpty(geometry_column)) {

            EditableFeatureAttributeDescriptor attr = (EditableFeatureAttributeDescriptor) fType.get(geometry_column);

            if (attr != null ) {

                if( attr.getType() != DataTypes.GEOMETRY ) {

                    attr.setDataType(DataTypes.GEOMETRY);

                }

                GeometryManager geommgr = GeometryLocator.getGeometryManager();

                GeometryType gt;

                try {

                    gt = geommgr.getGeometryType(

                            CSVStoreParameters.getGeometryType(parameters),

                            CSVStoreParameters.getGeometrySubType(parameters)

                    );

                    attr.setGeometryType(gt);

                } catch (Exception e) {

                    LOGGER.warn("Can't set geometry type for the calculated field in CSV file '" + fullFileName + "'.", e);

                }

                fType.setDefaultGeometryAttributeName(geometry_column);

            }

        }

        return all_fields_declare_type;

    }

}

/**

 * gvSIG. Desktop Geographic Information System.

 *

 * Copyright (C) 2007-2013 gvSIG Association.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 3

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

 * MA  02110-1301, USA.

 *

 * For any additional information, do not hesitate to contact us

 * at info AT gvsig.com, or visit our website www.gvsig.com.

 */

package org.gvsig.fmap.dal.store.csv;

import java.util.ArrayList;

import java.util.List;

import org.gvsig.fmap.dal.DALFileLibrary;

import org.gvsig.fmap.dal.DALFileLocator;

import org.gvsig.fmap.dal.DALLibrary;

import org.gvsig.fmap.dal.DALLocator;

import org.gvsig.fmap.dal.FileHelper;

import org.gvsig.fmap.dal.spi.DataManagerProviderServices;

import org.gvsig.metadata.exceptions.MetadataException;

import org.gvsig.tools.library.AbstractLibrary;

import org.gvsig.tools.library.LibraryException;

public class CSVLibrary extends AbstractLibrary {

    @Override

    public void doRegistration() {

        registerAsServiceOf(DALLibrary.class);

        require(DALFileLibrary.class);

    }

    @Override

    protected void doInitialize() throws LibraryException {

    }

    @Override

    protected void doPostInitialize() throws LibraryException {

        List<Throwable> exs = new ArrayList<Throwable>();

        FileHelper.registerParametersDefinition(

                CSVStoreParameters.PARAMETERS_DEFINITION_NAME,

                CSVStoreParameters.class, "CSVParameters.xml");

        FileHelper.registerParametersDefinition(

                CSVNewStoreParameters.PARAMETERS_DEFINITION_NAME,

                CSVNewStoreParameters.class, "CSVParameters.xml");

        try {

        } catch (Exception e) {

            exs.add(e);

        }

        try {

            FileHelper.registerMetadataDefinition(

                    CSVStoreProvider.METADATA_DEFINITION_NAME,

                    CSVStoreProvider.class, "CSVMetadata.xml");

        } catch (MetadataException e) {

            exs.add(e);

        }

        DataManagerProviderServices dataman = (DataManagerProviderServices) DALLocator

                .getDataManager();

        try {

            if (!dataman.getStoreProviders().contains(CSVStoreProvider.NAME)) {

                dataman.registerStoreProviderFactory(new CSVStoreProviderFactory(CSVStoreProvider.NAME, CSVStoreProvider.DESCRIPTION));

            }

        } catch (RuntimeException e) {

            exs.add(e);

        }

        try {

            DALFileLocator.getFilesystemServerExplorerManager()

                    .registerProvider(CSVStoreProvider.NAME,

                            CSVStoreProvider.DESCRIPTION,

                            CSVFilesystemServerProvider.class);

        } catch (RuntimeException e) {

            exs.add(e);

        }

        if (exs.size() > 0) {

            throw new LibraryException(this.getClass(), exs);

        }

    }

}

package org.gvsig.fmap.dal.store.csv;

import java.io.File;

import java.io.IOException;

import java.io.RandomAccessFile;

import java.nio.ByteBuffer;

import java.nio.MappedByteBuffer;

import java.nio.channels.FileChannel;

import java.sql.Timestamp;

import java.util.AbstractList;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.io.IOUtils;

import org.gvsig.fmap.dal.store.csv.RecordsFile.Record;

import static org.gvsig.fmap.dal.store.csv.RecordsFile.TYPE_BYTE;

import static org.gvsig.fmap.dal.store.csv.RecordsFile.TYPE_STRING;

import static org.gvsig.fmap.dal.store.csv.RecordsFile.TYPE_TIMESTAMP;

/**

 *

 * @author jjdelcerro

 */

public class RecordsFileImpl 

        extends AbstractList<Record>

        implements RecordsFile 

    {

    public static class RecordImpl implements Record {

        private final RecordType type;

        private final ByteBuffer buffer;

        public RecordImpl(RecordType type) {

            this.type = type;

            this.buffer = ByteBuffer.allocate(this.type.getSize());

        }

        @Override

        public RecordType getType() {

            return this.type;

        }

        @Override

        public byte[] getBytes() {

            return this.buffer.array();

        }

        @Override

        public ByteBuffer getBuffer() {

            return this.buffer;

        }

        @Override

        public void setBytes(byte[] bytes) {

            this.buffer.position(0);

            this.buffer.put(bytes);

        }

        @Override

        public int getInt(int n) {

            this.buffer.position(this.type.getFieldOffset(n));

            return this.buffer.getInt();

        }

        @Override

        public long getLong(int n) {

            this.buffer.position(this.type.getFieldOffset(n));

            return this.buffer.getLong();

        }

        @Override

        public double getDouble(int n) {

            this.buffer.position(this.type.getFieldOffset(n));

            return this.buffer.getDouble();

        }

        @Override

        public Timestamp getTimestamp(int n) {

            this.buffer.position(this.type.getFieldOffset(n));

            long l = this.buffer.getLong();

            return new Timestamp(l);

        }

        @Override

        public String getString(int n) {

            this.buffer.position(this.type.getFieldOffset(n));

            short sz = this.buffer.getShort();

            char[] chars = new char[sz];

            for (int i = 0; i < sz; i++) {

                chars[i] = this.buffer.getChar(i);

            }

            return new String(chars);

        }

        @Override

        public byte[] getBytes(int n) {

            this.buffer.position(this.type.getFieldOffset(n));

            int sz = this.buffer.getInt();

            byte[] bytes = new byte[sz];

            for (int i = 0; i < sz; i++) {

                bytes[i] = this.buffer.get(i);

            }

            return bytes;

        }

        @Override

        public int getByte(int n) {

            this.buffer.position(this.type.getFieldOffset(n));

            return this.buffer.get();

        }

        @Override

        public int getShort(int n) {

            this.buffer.position(this.type.getFieldOffset(n));

            return this.buffer.getShort();

        }

        @Override

        public void setByte(int n, byte v) {

            this.buffer.position(this.type.getFieldOffset(n));

            this.buffer.put(v);

        }

        @Override

        public void setShort(int n, short v) {

            this.buffer.position(this.type.getFieldOffset(n));

            this.buffer.putShort(v);

        }

        @Override

        public void setInt(int n, int v) {

            this.buffer.position(this.type.getFieldOffset(n));

            this.buffer.putInt(v);

        }

        @Override

        public void setLong(int n, long v) {

            this.buffer.position(this.type.getFieldOffset(n));

            this.buffer.putLong(v);

        }

        @Override

        public void setDouble(int n, double v) {

            this.buffer.position(this.type.getFieldOffset(n));

            this.buffer.putDouble(v);

        }

        @Override

        public void setTimestamp(int n, Timestamp v) {

            this.buffer.position(this.type.getFieldOffset(n));

            this.buffer.putLong(v.getTime());

        }

        @Override

        public void setString(int n, String v) {

            this.buffer.position(this.type.getFieldOffset(n));

            this.buffer.putShort((short) v.length());

            for (int i = 0; i < v.length(); i++) {

                this.buffer.putChar(v.charAt(i));

            }

        }

        @Override

        public void setBytes(int n, byte[] v) {

            this.buffer.position(this.type.getFieldOffset(n));

            this.buffer.putInt(v.length);

            for (int i = 0; i < v.length; i++) {

                this.buffer.put(v[i]);

            }

        }

    }

    public static class RecordTypeImpl implements RecordType {

        private static class FieldType {

            public byte type;

            public int size;

            public int offset;

        }

        private final List<FieldType> fields;

        private int size;

        public RecordTypeImpl() {

            this.fields = new ArrayList<>();

            this.size = -1;

        }

        public void add(int type, int size) {

            FieldType field = new FieldType();

            field.size = size;

            field.type = (byte) type;

            this.size = -1;

            field.offset = this.getSize();

            this.fields.add(field);

            this.size = -1;

            this.size = this.getSize();

        }

        @Override

        public int getSize() {

            if( this.size<0 ) {

                int sz = 0;

                for (FieldType field : fields) {

                    switch(field.type) {

                        case TYPE_BYTE:

                            sz += 1;

                            break;

                        case TYPE_SHORT:

                            sz += 2;

                            break;

                        case TYPE_INTEGER:

                            sz += 4;

                            break;

                        case TYPE_TIMESTAMP:

                        case TYPE_LONG:

                            sz += 8;

                            break;

                        case TYPE_DOUBLE:

                            sz += 8;

                            break;

                        case TYPE_STRING:

                            sz += 2 + (field.size*2);

                            break;

                        case TYPE_BYTES:

                            sz += 4 + field.size;

                            break;

                    }

                }

                this.size = sz;

            }

            return this.size;

        }

        @Override

        public int getFieldCount() {

            return this.fields.size();

        }

        @Override

        public int getFieldType(int n) {

            return this.fields.get(n).type;

        }

        @Override

        public int getFieldSize(int n) {

            return this.fields.get(n).size;

        }

        @Override

        public int getFieldOffset(int n) {

            return this.fields.get(n).offset;

        }

        @Override

        public Record createRecord() {

            this.size = -1;

            this.size = this.getSize();

            return new RecordImpl(this);

        }

        @Override

        public byte[] toBytes() {

            ByteBuffer buffer = ByteBuffer.allocate(this.fields.size()*(1+4));

            for (FieldType field : fields) {

                buffer.put(field.type);

                buffer.putInt(field.size);

            }

            return buffer.array();

        }

        public static RecordType from(ByteBuffer bytes) {

            RecordTypeImpl recordType = new RecordTypeImpl();

            bytes.position(0);

            int len = bytes.limit()/(1+4);

            for (int i = 0; i < len; i++) {

                int type = bytes.get();

                int size = bytes.getInt();

                recordType.add(type, size);

            }

            return recordType;

        }

        public static RecordType from(byte[] bytes) {

            ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.length);

            buffer.put(bytes);

            return from(buffer);

        }

    }

    public static class RecordTypeBuilderImpl implements RecordTypeBuilder {

        private final RecordTypeImpl recordType;

        public RecordTypeBuilderImpl() {

            this.recordType = new RecordTypeImpl();

        }

        @Override

        public RecordTypeBuilder addbyte() {

            this.recordType.add(TYPE_BYTE, 0);

            return this;

        }

        @Override

        public RecordTypeBuilder addShort() {

            this.recordType.add(TYPE_SHORT, 0);

            return this;

        }

        @Override

        public RecordTypeBuilder addInteger() {

            this.recordType.add(TYPE_INTEGER, 0);

            return this;

        }

        @Override

        public RecordTypeBuilder addLong() {

            this.recordType.add(TYPE_LONG, 0);

            return this;

        }

        @Override

        public RecordTypeBuilder addDouble() {

            this.recordType.add(TYPE_DOUBLE, 0);

            return this;

        }

        @Override

        public RecordTypeBuilder addTimestamp() {

            this.recordType.add(TYPE_TIMESTAMP, 0);

            return this;

        }

        @Override

        public RecordTypeBuilder addString(int size) {

            this.recordType.add(TYPE_STRING, size);

            return this;

        }

        @Override

        public RecordTypeBuilder addBytes(int size) {

            this.recordType.add(TYPE_BYTES, size);

            return this;

        }

        @Override

        public RecordType build() {

            return this.recordType;

        }

        public static RecordTypeBuilder recordTypeBuilder() {

            return new RecordTypeBuilderImpl();

        }

    }

    private static final int HEADER_SIZE = 1024;

    private RandomAccessFile raf;

    private MappedByteBuffer fileByteBuffer;

    private long sz;

    private int header_size;

    private int type_size;

    private RecordType recordType;

    public RecordsFileImpl() {

    }

    @SuppressWarnings("OverridableMethodCallInConstructor")

    public RecordsFileImpl(File f) throws IOException {

        this.open(f);

    }

    @SuppressWarnings("OverridableMethodCallInConstructor")

    public RecordsFileImpl(RandomAccessFile raf) throws IOException {

        this.open(raf);

    }

    @Override

    public RecordType getRecordType() {

        return this.recordType;

    }

    @Override

    public void open(File f) throws IOException {

        RandomAccessFile theRaf = new RandomAccessFile(f,"r");

        this.open(theRaf);

    }

    @Override

    public void open(RandomAccessFile raf) throws IOException {

        this.raf = raf;

        this.closeByteBuffer();

        this.raf.seek(0);

        this.raf.readLong(); //For future use

        this.raf.readLong(); //For future use

        this.header_size = (int) this.raf.readLong(); //Header size

        this.type_size = (int) this.raf.readLong(); //Record length

        byte[] recordTypeBytes = new byte[this.type_size];

        this.raf.read(recordTypeBytes);

        this.recordType = RecordType.from(recordTypeBytes);

        this.sz = (this.raf.length()-this.header_size) / this.recordType.getSize();

        // TODO: meter en la cabecera el numero de registros para poder escribir datos al final

    }

    @Override

    public void create(File f, RecordType recordType) throws IOException {

        RandomAccessFile theRaf = new RandomAccessFile(f,"rw");

        this.create(theRaf,recordType);

    }

    @Override

    public void create(RandomAccessFile raf, RecordType recordType) throws IOException {

        this.raf = raf;

        this.recordType = recordType;

        this.header_size = HEADER_SIZE;

        this.closeByteBuffer();

        this.sz = 0;

        byte[] recordTypeBytes = this.recordType.toBytes();

        this.raf.seek(0);

        this.raf.writeLong(0); //For future use

        this.raf.writeLong(0); //For future use

        this.raf.writeLong(this.header_size); //Header size

        this.raf.writeLong(recordTypeBytes.length); //Record length

        this.raf.write(recordTypeBytes);

        this.raf.setLength(this.header_size);

    }

    @Override

    public void close() throws IOException {

        this.closeByteBuffer();

        IOUtils.closeQuietly(this.raf);

        this.raf = null;

        this.sz = -1;

    }

    @Override

    public boolean isOpen() {

        return this.raf!=null;

    }

    @Override

    public Record get(int position) {

        position = checkIndex(position);

        Record record = this.recordType.createRecord();

        this.getByteBuffer().position(this.header_size+(position*this.recordType.getSize()));

        this.getByteBuffer().get(record.getBytes());

        return record;

    }

    @Override

    public Record get64(long position) {

        position = checkIndex(position);

        Record record = this.recordType.createRecord();

        this.getByteBuffer().position((int) (this.header_size+(position*this.recordType.getSize())));

        this.getByteBuffer().get(record.getBytes());

        return record;

    }

    @Override

    public int size() {

        return (int) this.sz;

    }

    @Override

    public long size64() {

        return this.sz;

    }

    @Override

    public Record set(int position, Record record) {

        try {

            closeByteBuffer();

            position = checkIndex(position);

            this.raf.seek(this.header_size + (position * this.recordType.getSize()));

            this.raf.write(record.getBytes());

            return record;

        } catch (IOException ex) {

            throw new RuntimeException("Can't add record", ex);

        }

    }

    private int checkIndex(long index) {

        if (this.raf == null) {

            throw new IllegalStateException("Index not open");

        }

        if( index < 0 ) {

            index = ((int)this.sz) + index;

        }

        if( index<0 || index>=this.sz) {

            throw new IllegalArgumentException("Index out of range ("+index+")");

        }

        return (int) index;

    }

    private MappedByteBuffer getByteBuffer() {

        try {

            if (this.fileByteBuffer == null) {

                this.fileByteBuffer = this.raf.getChannel().map(

                        FileChannel.MapMode.READ_ONLY,

                        0,

                        this.raf.length()

                );

            }

            return this.fileByteBuffer;

        } catch (IOException ex) {

            throw new RuntimeException("Can't add record", ex);

        }

    }

    private void closeByteBuffer() {

        if(this.fileByteBuffer != null){

            this.fileByteBuffer.force();

            this.fileByteBuffer = null;

        }

    }

    @Override

    public boolean add(Record record) {

        try {

            this.raf.seek(this.raf.length());

            this.raf.write(record.getBytes());

            this.closeByteBuffer();

            this.sz++;

            return true;

        } catch (IOException ex) {

            throw new RuntimeException("Can't add record", ex);

        }

    }

}

package org.gvsig.fmap.dal.store.csv;

import java.io.File;

import java.io.FileInputStream;

import java.io.InputStreamReader;

import java.io.Reader;

import java.nio.charset.Charset;

import java.nio.charset.StandardCharsets;

import java.util.ArrayList;

import java.util.Collections;

import java.util.HashMap;

import java.util.HashSet;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.LinkedHashSet;

import java.util.List;

import java.util.Map;

import java.util.Set;

import javax.xml.parsers.SAXParser;

import javax.xml.parsers.SAXParserFactory;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.lang3.mutable.MutableLong;

import org.gvsig.tools.namestranslator.NamesTranslator;

import org.xml.sax.Attributes;

import org.xml.sax.InputSource;

import org.xml.sax.Locator;

import org.xml.sax.SAXException;

import org.xml.sax.helpers.DefaultHandler;

public class TestXML {

    private static final String XMLFILE1 = "/home/jjdelcerro/datos/geodata/vector/sigpac/2018/Declaracion.xml";

    private static final String XMLFILE2 = "/home/jjdelcerro/datos/geodata/vector/ARENA2/quincenas-0/TV_03_2019_01_Q1/victimas.xml";

    private static final String XMLFILE_BIG = "/home/jjdelcerro/datos/geodata/vector/RSUPAC/2020/BDA_RSU_PAC20_1713052020_001.XML";

    public static void main(String[] args) throws Exception {

        TestXML t = new TestXML();

        List<String> tags = new ArrayList(t.extractTags(XMLFILE_BIG));

        Collections.sort(tags);

//        System.out.println("Tags: -------------------------");

//        for (String tag : tags) {

//            System.out.println(tag);

//        }

        Map<String,List<String>> tables = new LinkedHashMap<>();

        Map<String,Set<String>> tables_tmp = new LinkedHashMap<>();

        for (String tag1 : tags) {

            String tableName = tag1;

            Set fields = new HashSet<>();

            for (String tag2 : tags) {

                if( tableName.equals(tag2) ) {

                    continue;

                }

                if( tag2.startsWith(tableName) ) {

                    String fieldName = tag2.substring(tableName.length()+1);

                    if( !fieldName.contains("/") ) {

                        fields.add(fieldName);

                    }

                }

            }

            if( !fields.isEmpty() ) {

                tables_tmp.put(tableName,fields);

            }

        }

        Map<String,String> tableNames = new HashMap<>();

//        NamesTranslator nt = NamesTranslator.createTrimTranslator(45);

        for (Map.Entry<String, Set<String>> entry : tables_tmp.entrySet()) {

            String fullTableName = entry.getKey();

            String tableName = FilenameUtils.getBaseName(fullTableName);

//            int i = nt.addSource(fullTableName);

//            tableNames.put(tableName,nt.getTranslation(i));

            tableNames.put(fullTableName,tableName);

        }

        for (Map.Entry<String, Set<String>> entry : tables_tmp.entrySet()) {

            String tableName = entry.getKey();

            List<String> fields = new ArrayList<>();

            String[] fieldkeys = tableName.split("/");

            for (int i = 0; i < fieldkeys.length; i++) {

                String fieldkey = fieldkeys[i];

                if( i==fieldkeys.length-1 ) {

                    fields.add("$ID_"+fieldkey+", PK");

                } else {

                    fields.add("$ID_"+fieldkey+", FK");

                }

            }

            for (String fieldName : entry.getValue()) {

                if( tables_tmp.containsKey(tableName+"/"+fieldName) ) {

                    fields.add("$ID_"+fieldName+" FK");

                } else {

                    fields.add(fieldName);

                }

            }            

            Collections.sort(fields);

            tables.put(tableName, fields);

        }

        int n = 1;

        for (Map.Entry<String, List<String>> entry : tables.entrySet()) {

            String tableName = entry.getKey();

            List<String> fields = entry.getValue();

            System.out.println("TABLE "+tableNames.get(tableName)+" ("+n+", "+tableName+")");

            for (String field : fields) {

                System.out.println("    "+field);

            }

            n++;

        }

//        List<List<String>> records = t.getRecords1(

//                XMLFILE1, 

//                "DECLARACION/LINEA_DECLARACION", 

//                ListBuilder.create(

//                    "DECLARACION/LINEA_DECLARACION/PROV",

////                    "DECLARACION/LINEA_DECLARACION/MUN_INE",

////                    "DECLARACION/LINEA_DECLARACION/MUN_CAT",

////                    "DECLARACION/LINEA_DECLARACION/AGREGADO",

//                    "DECLARACION/LINEA_DECLARACION/ZONA",

//                    "DECLARACION/LINEA_DECLARACION/POLIGONO",

//                    "DECLARACION/LINEA_DECLARACION/PARCELA",

//                    "DECLARACION/LINEA_DECLARACION/RECINTO",

//                    "DECLARACION/LINEA_DECLARACION/PARCELA_AGRICOLA",

//                    "DECLARACION/LINEA_DECLARACION/CULTIVO",

////                    "DECLARACION/LINEA_DECLARACION/WKT",

//                    "DECLARACION/LINEA_DECLARACION/DN_SURFACE",

//                    "DECLARACION/LINEA_DECLARACION/SUPERFICIE_DECLARADA"

////                    "DECLARACION/LINEA_DECLARACION/FC_ALMENDROS",

////                    "DECLARACION/LINEA_DECLARACION/FC_ALGARROBOS",

////                    "DECLARACION/LINEA_DECLARACION/FC_AVELLANOS",

////                    "DECLARACION/LINEA_DECLARACION/FC_NOGALES",

////                    "DECLARACION/LINEA_DECLARACION/FC_PISTACHOS",

////                    "DECLARACION/LINEA_DECLARACION/FC_TOTAL"

//                )