Application: gvSIG desktop

/**

 * gvSIG. Desktop Geographic Information System.

 *

 * Copyright (C) 2007-2013 gvSIG Association.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 3

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

 * MA  02110-1301, USA.

 *

 * For any additional information, do not hesitate to contact us

 * at info AT gvsig.com, or visit our website www.gvsig.com.

 */

package org.gvsig.fmap.dal.store.dbf.utils;

import java.io.IOException;

import java.math.BigDecimal;

import java.nio.Buffer;

import java.nio.BufferOverflowException;

import java.nio.ByteBuffer;

import java.nio.MappedByteBuffer;

import java.nio.channels.FileChannel;

import java.nio.charset.Charset;

import java.util.Date;

import java.util.Iterator;

import org.apache.commons.lang3.StringUtils;

import org.gvsig.fmap.dal.DataTypes;

import org.gvsig.fmap.dal.exception.CloseException;

import org.gvsig.fmap.dal.exception.InitializeException;

import org.gvsig.fmap.dal.exception.UnsupportedEncodingException;

import org.gvsig.fmap.dal.exception.WriteException;

import org.gvsig.fmap.dal.feature.Feature;

import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.FeatureType;

import org.gvsig.tools.logger.FilteredLogger;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

/**

 * A DbaseFileReader is used to read a dbase III format file. The general use of

 * this class is: <CODE><PRE>

 * DbaseFileHeader header = ...

 * WritableFileChannel out = new FileOutputStream("thefile.dbf").getChannel();

 * DbaseFileWriter w = new DbaseFileWriter(header,out);

 * while ( moreRecords ) {

 *   w.write( getMyRecord() );

 * }

 * w.close();

 * </PRE></CODE> You must supply the <CODE>moreRecords</CODE> and

 * <CODE>getMyRecord()</CODE> logic...

 *

 * @author Ian Schneider

 */

public class DbaseFileWriter {

    private static final Logger LOGGER = LoggerFactory.getLogger(DbaseFileWriter.class);

    private final DbaseFileHeader header;

    private FieldFormatter formatter = new FieldFormatter();

    private FileChannel channel;

    private ByteBuffer buffer;

    private boolean headDrity = false;

    private ByteBuffer blank;

    private int blankSize;

    private Charset charset;

    private FilteredLogger logger;

    /**

     * Create a DbaseFileWriter using the specified header and writing to the

     * given channel.

     *

     * @param header The DbaseFileHeader to write.

     * @param out The Channel to write to.

     * @param isNew

     * @throws org.gvsig.fmap.dal.exception.InitializeException

     *

     *

     */

    public DbaseFileWriter(DbaseFileHeader header, FileChannel out,

            boolean isNew) throws InitializeException {

        this.header = header;

        this.channel = out;

        this.headDrity = isNew;

        this.setCharset(Charset.forName(header.mappingEncoding(header.getCharsetName())));

        init();

    }

    private void init() throws InitializeException {

        try {

            if (this.channel.size() < this.header.getHeaderLength()) {

                this.writeHeader();

            }

            buffer = ByteBuffer.allocateDirect(header.getRecordLength());

        } catch (Exception e) {

            throw new InitializeException("DBF Writer", e);

        }

    }

    private void write() throws WriteException {

        ((Buffer) buffer).position(0);

        int r = buffer.remaining();

        try {

            while ((r -= channel.write(buffer)) > 0) {

                // do nothing

            }

        } catch (IOException e) {

            throw new WriteException("DBF Writer", e);

        }

    }

    private void writeHeader() throws WriteException {

        try {

            channel.position(0);

            header.write(channel);

        } catch (IOException e) {

            throw new WriteException("DBF Writer", e);

        }

    }

    /**

     * Write a single dbase record.

     *

     * @param feature

     * @throws UnsupportedEncodingException

     * @throws WriteException

     */

    public void append(Feature feature) throws WriteException,

            UnsupportedEncodingException {

        this.fillBuffer(feature);

        try {

            this.moveToEOF();

        } catch (IOException e) {

            throw new WriteException("DbaseFileWriter", e);

        }

        this.header.setNumRecords(this.header.getNumRecords() + 1);

        write();

        this.headDrity = true;

    }

    private void fillBuffer(Feature feature)

            throws UnsupportedEncodingException, WriteException {

        FeatureType featureType = feature.getType();

        try {

            ((Buffer) buffer).position(0);

            // put the 'not-deleted' marker

            buffer.put((byte) ' ');

            @SuppressWarnings("unchecked")

            Iterator<FeatureAttributeDescriptor> iterator

                    = featureType.iterator();

            while (iterator.hasNext()) {

                FeatureAttributeDescriptor fad = iterator.next();

                if (fad.isComputed()) {

                    continue;

                }

                if (fad.getName().length() > DbaseFile.MAX_FIELD_NAME_LENGTH) {

                    throw new FieldNameTooLongException(

                            "DBF file", fad.getName());

                }

                int type = fad.getType();

                if (type == DataTypes.GEOMETRY) {

                    continue;

                }

                encodeField(fad, feature);

            }

        } catch (Exception e) {

            throw new WriteException("DbaseFileWriter", e);

        }

    }

    private void moveToEOF() throws IOException {

        this.moveTo(this.header.getNumRecords());

    }

    private void moveTo(long numReg) throws IOException {

        // if (!(channel instanceof FileChannel)) {

        // throw new IOException(

        // "DbaseFileWriterNIO: channel is not a FileChannel. Cannot position properly");

        // }

        long newPos

                = header.getHeaderLength() + numReg * header.getRecordLength();

        if (this.channel.position() != newPos) {

            this.channel.position(newPos);

        }

    }

    /**

     * Write a single dbase record. Useful to update a dbf.

     *

     * @param feature

     * @param numReg

     * @throws WriteException

     * @throws UnsupportedEncodingException

     */

    public void update(Feature feature, long numReg) throws WriteException,

            UnsupportedEncodingException {

        this.fillBuffer(feature);

        try {

            this.moveTo(numReg);

        } catch (IOException e) {

            throw new WriteException("DbaseFileWriter", e);

        }

        write();

    }

    private void encodeField(FeatureAttributeDescriptor attr, Feature feature) throws java.io.UnsupportedEncodingException, UnsupportedEncodingException {

        if (attr == null) {

            throw new NullPointerException("attr is NULL");

        }

        if (feature == null) {

            throw new NullPointerException("feature is NULL");

        }

        int fieldLen = -1;

        try {

            DbaseFieldDescriptor descriptor = this.header.getFieldDescription(attr.getName());

            int type = attr.getType();

            fieldLen = descriptor.getSize();

            String fieldString;

//        if( buffer.position()!=descriptor.getOffsetInRecord() ) {

//            throw new RuntimeException("Encoding field '"+descriptor.getName()+"' found an incorrect offset.");

//        }

            String fieldName = attr.getName();

            if (feature.isNull(attr.getIndex())) {

                safeEncode(fieldName, " ", fieldLen, false);

                return;

            }

            if (DataTypes.BOOLEAN == type) {

                boolean b = feature.getBoolean(attr.getIndex());

                safeEncode(fieldName, b ? "T" : "F", 1, true);

            } else if (DataTypes.TIME == type) {

                Date date = feature.getTime(attr.getIndex());

                fieldString = formatter.formatTime(date);

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.TIMESTAMP == type) {

                Date date = feature.getTimestamp(attr.getIndex());

                fieldString = formatter.formatTimestamp(date);

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.DATE == type) {

                Date date = feature.getDate(attr.getIndex());

                fieldString = formatter.formatDate(date);

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.DECIMAL == type) {

                BigDecimal n = feature.getDecimal(attr.getIndex());

                fieldString = formatter.format(n, fieldLen);

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.DOUBLE == type) {

                double n = feature.getDouble(attr.getIndex());

                fieldString = formatter.format(n, fieldLen, descriptor.getScale());

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.FLOAT == type) {

                float n = feature.getFloat(attr.getIndex());

                fieldString = formatter.format(n, fieldLen, descriptor.getScale());

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.LONG == type) {

                long l = feature.getLong(attr.getIndex());

                fieldString = formatter.format(l, fieldLen);

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.INT == type) {

                int n = feature.getInt(attr.getIndex());

                fieldString = formatter.format(n, fieldLen);

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.BYTE == type) {

                int n = feature.getInt(attr.getIndex());

                fieldString = formatter.format(n, fieldLen);

                safeEncode(fieldName, fieldString, fieldLen, false);

            } else if (DataTypes.STRING == type) {

                String s = feature.getString(attr.getIndex());

                safeEncode(fieldName, StringUtils.defaultIfEmpty(s, ""), fieldLen, true);

            } else {

                // Si no conocemos el tipo intentamos guardarlo como un string

                String s = feature.getString(attr.getIndex());

                safeEncode(fieldName, StringUtils.defaultIfEmpty(s, ""), fieldLen, true);

            }

        } catch (Exception ex) {

            throw new RuntimeException("Can't encode field '" + attr.getName() + "' with size "+fieldLen, ex);

        }

    }

    /**

     * Returns a safely padded (and potentially truncated) string

     *

     * This may truncate some record, but it is required to ensure that the

     * field limit is not overflowed when using variable-length charsets such as

     * UTF-8.

     *

     * @throws UnsupportedEncodingException

     */

    private void safeEncode(String fieldName, String in, int limit, boolean rightPadding) throws UnsupportedEncodingException {

        try {

            byte[] encodedString = in.getBytes(this.charset);

            if (encodedString.length > limit) {

                this.getLogger().error("'"+fieldName+"' field size ["+encodedString.length+"] exceeds limit "+limit);

                // too long, truncating

                /*

                             * The block code bellow is equivalent to this simple code

                             * fragment:

                    if (rightPadding) {

                            in = in.substring(0, in.length()-1);

                            encodedString = in.getBytes(charset);

                    }

                    else {

                            in.substring(1, in.length());

                            encodedString = in.getBytes(charset);

                    }

                    However, the implemented algorithm has a much better performance

                    for the average and worst cases (when the input string has a lot

                    of multibyte characters), while keeping a good performance

                    for the best case (when all the characters in the input string

                    can be represented as single bytes using the selected charset).

                    The general strategy is to compute the deviation from the

                    required maximum number of bytes (limit) and the actual number

                    of bytes of the encoded String.

                    Then, we use this deviation to estimate the amount of characters

                    to truncate, based on the average factor of bytes per char in the

                    input string.

                    We truncate the string using this approach until the deviation

                    gets stable.

                    Finally, as we should be close enough to the right truncation position,

                    we increment/decrement the truncated string by only 1 character, to

                    ensure we truncate in the exact position. 

                 */

                String str = in;

                int estimatedDiff, deviation;

                int deviationPrev;

                double ratio;

                byte[] encodedChar;

                int truncatePos = 0;

                deviation = encodedString.length - limit;

                deviationPrev = deviation - 1;

                while (Math.abs(deviation) > Math.abs(deviationPrev) && str.length() > 0) {

                    ratio = ((double) encodedString.length) / ((double) str.length());

                    // apply the estimated diff, ensuring it is at least >= 1.0 in absolute value

                    estimatedDiff = Math.max((int) (((double) deviation) / ratio), (int) (Math.signum(deviation) * 1));

                    // too long, truncating

                    if (rightPadding) {

                        truncatePos = Math.max(str.length() - estimatedDiff, 0);

                        str = in.substring(0, truncatePos);

                    } else {

                        truncatePos = Math.max(truncatePos + estimatedDiff, 0);

                        str = in.substring(truncatePos);

                    }

                    encodedString = str.getBytes(charset);

                    deviationPrev = deviation;

                    deviation = encodedString.length - limit;

                }

                // now we are close enough, get the exact position for truncating

                while (encodedString.length > limit) {

                    // too long, truncating

                    //                                      System.out.println("truncating");

                    if (rightPadding) {

                        str = in.substring(0, str.length() - 1);

                    } else {

                        truncatePos = truncatePos + 1;

                        str = in.substring(truncatePos);

                    }

                    encodedString = str.getBytes(charset);

                }

                while (encodedString.length < limit && str.length() < in.length()) {

                    // Extend if necessary:

                    // 1 - Get the length in bytes of the next char

                    // 2 - Add the char to the substring if we are still within the limits 

                    //                                      System.out.println("extending");

                    if (rightPadding) {

                        encodedChar = in.substring(str.length(), str.length() + 1).getBytes(charset);

                    } else {

                        encodedChar = in.substring(truncatePos - 1, truncatePos).getBytes(charset);

                        //                                              System.out.println(encodedChar);

                        //                                              System.out.println(encodedChar.length);

                        //                                              System.out.println(testStrings[i].substring(truncatePos-1, truncatePos));

                    }

                    //                                      System.out.println(testStrings[i].substring(in.length(), in.length()+1));

                    if ((encodedString.length + encodedChar.length) > limit) {

                        // one more char would overflow the limit

                        break;

                    }

                    // too short, extending

                    if (rightPadding) {

                        str = in.substring(0, str.length() + 1);

                    } else {

                        truncatePos = truncatePos - 1;

                        str = in.substring(truncatePos);

                    }

                    encodedString = str.getBytes(charset);

                }

            }

            if (rightPadding) {

                buffer.put(encodedString);

            }

            if (encodedString.length < limit) {

                // too short, padding

                int i = encodedString.length;

                while (i < limit) {

                    ((Buffer) blank).position(0);

                    buffer.put(blank);

                    i = i + blankSize;

                }

                if (i > limit) {

                    // Might happen for instance if charset is UTF16 and the

                    // limit of characters in the field is an odd number

                    throw new UnsupportedEncodingException(new Exception("Impossible to encode this DBF using the selected charset"));

                }

            }

            if (!rightPadding) {

                buffer.put(encodedString);

            }

        } catch (BufferOverflowException exc) {

            // Might happen for instance if charset is UTF16 and the

            // limit of characters in the field is an odd number

            throw new UnsupportedEncodingException(exc);

        }

    }

    /**

     * Release resources associated with this writer. <B>Highly recommended</B>

     *

     * @throws CloseException

     */

    public void close() throws CloseException {

        // IANS - GEOT 193, bogus 0x00 written. According to dbf spec, optional

        // eof 0x1a marker is, well, optional. Since the original code wrote a

        // 0x00 (which is wrong anyway) lets just do away with this :)

        // - produced dbf works in OpenOffice and ArcExplorer java, so it must

        // be okay.

        // buffer.position(0);

        // buffer.put((byte) 0).position(0).limit(1);

        // write();

        if (headDrity) {

            try {

                this.writeHeader();

            } catch (WriteException e) {

                throw new CloseException("DbaseFileWriter", e);

            }

        }

        try {

            channel.close();

        } catch (IOException e) {

            throw new CloseException("DBF Writer", e);

        }

        if (buffer instanceof MappedByteBuffer) {

            // NIOUtilities.clean(buffer);

        }

        buffer = null;

        channel = null;

        formatter = null;

    }

    public void setCharset(Charset charset) {

        this.charset = charset;

        blank = charset.encode(" ");

        blankSize = blank.limit();

    }

    protected FilteredLogger getLogger() {

        if(this.logger == null){

            this.logger = new FilteredLogger(LOGGER, "DbaseFileWriter", 2000L);

        }

        return this.logger;

    }

}