Revision 44936
| trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/AutomaticDetectionOfTypes.java | ||
|---|---|---|
| 1 | 1 |
package org.gvsig.fmap.dal.store.csv; |
| 2 | 2 |
|
| 3 | 3 |
import java.io.IOException; |
| 4 |
import java.math.BigDecimal; |
|
| 4 | 5 |
import java.net.URL; |
| 5 | 6 |
import java.util.ArrayList; |
| 6 | 7 |
import java.util.List; |
| ... | ... | |
| 22 | 23 |
|
| 23 | 24 |
public List<String> nextRowValues(); |
| 24 | 25 |
} |
| 26 |
|
|
| 27 |
public interface DetectedValue {
|
|
| 28 |
public int getType(); |
|
| 29 |
public int getDisplaySize(); |
|
| 30 |
public int getPrecision(); |
|
| 31 |
public int getScale(); |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
private static class DetectedValueImpl implements DetectedValue {
|
|
| 25 | 35 |
|
| 36 |
private int type; |
|
| 37 |
private int displaySize; |
|
| 38 |
private int precision; |
|
| 39 |
private int scale; |
|
| 40 |
|
|
| 41 |
@Override |
|
| 42 |
public int getType() {
|
|
| 43 |
return this.type; |
|
| 44 |
} |
|
| 45 |
|
|
| 46 |
@Override |
|
| 47 |
public int getDisplaySize() {
|
|
| 48 |
return this.displaySize; |
|
| 49 |
} |
|
| 50 |
|
|
| 51 |
@Override |
|
| 52 |
public int getPrecision() {
|
|
| 53 |
return this.precision; |
|
| 54 |
} |
|
| 55 |
|
|
| 56 |
@Override |
|
| 57 |
public int getScale() {
|
|
| 58 |
return this.scale; |
|
| 59 |
} |
|
| 60 |
|
|
| 61 |
} |
|
| 62 |
|
|
| 26 | 63 |
private static class PossibleDataType {
|
| 27 | 64 |
|
| 28 | 65 |
public boolean possibleInt = true; |
| ... | ... | |
| 50 | 87 |
} |
| 51 | 88 |
|
| 52 | 89 |
@SuppressWarnings({"UseSpecificCatch", "ResultOfObjectAllocationIgnored"})
|
| 53 |
public int[] detect(int columns, Rows rows, boolean isFirstLineHeader, Locale locale) throws IOException {
|
|
| 90 |
public DetectedValue[] detect(int columns, Rows rows, boolean isFirstLineHeader, Locale locale) throws IOException {
|
|
| 54 | 91 |
List<PossibleDataType> possibleDataTypes; |
| 55 |
int[] types = null;
|
|
| 92 |
DetectedValueImpl[] detectedValues = new DetectedValueImpl [columns];
|
|
| 56 | 93 |
|
| 57 | 94 |
int lineno = 0; |
| 58 | 95 |
try {
|
| ... | ... | |
| 63 | 100 |
possibleDataTypes = new ArrayList<>(columns); |
| 64 | 101 |
for (int i = 0; i < columns; i++) {
|
| 65 | 102 |
possibleDataTypes.add(new PossibleDataType()); |
| 103 |
detectedValues[i] = new DetectedValueImpl(); |
|
| 66 | 104 |
} |
| 67 | 105 |
if (locale == null) {
|
| 68 | 106 |
locale = Locale.getDefault(); |
| ... | ... | |
| 86 | 124 |
possibleDataTypes.add(new PossibleDataType()); |
| 87 | 125 |
} |
| 88 | 126 |
String rawvalue = row.get(i); |
| 127 |
if( rawvalue == null ) {
|
|
| 128 |
continue; |
|
| 129 |
} |
|
| 89 | 130 |
PossibleDataType possibleDataType = possibleDataTypes.get(i); |
| 131 |
DetectedValueImpl detectedValue = detectedValues[i]; |
|
| 132 |
int displaySize = rawvalue.length(); |
|
| 133 |
if( displaySize>detectedValue.displaySize ) {
|
|
| 134 |
detectedValue.displaySize = displaySize; |
|
| 135 |
} |
|
| 90 | 136 |
if (possibleDataType.possibleDecimal) {
|
| 91 | 137 |
try {
|
| 92 |
toDecimal.coerce(rawvalue, coercionContext); |
|
| 138 |
BigDecimal decimal = (BigDecimal) toDecimal.coerce(rawvalue, coercionContext);
|
|
| 93 | 139 |
possibleDataType.possibleDecimal = true; |
| 140 |
if( decimal.scale() > detectedValue.scale ) {
|
|
| 141 |
detectedValue.scale = decimal.scale(); |
|
| 142 |
} |
|
| 143 |
if( decimal.precision()>detectedValue.precision ) {
|
|
| 144 |
detectedValue.precision = decimal.precision(); |
|
| 145 |
} |
|
| 94 | 146 |
} catch (Exception ex) {
|
| 95 | 147 |
possibleDataType.possibleDecimal = false; |
| 96 | 148 |
} |
| ... | ... | |
| 145 | 197 |
row = rows.nextRowValues(); |
| 146 | 198 |
} |
| 147 | 199 |
int n = 0; |
| 148 |
types = new int[possibleDataTypes.size()]; |
|
| 149 | 200 |
for (PossibleDataType possibleDataType : possibleDataTypes) {
|
| 150 | 201 |
if (possibleDataType.possibleInt) {
|
| 151 |
types[n++] = DataTypes.INT;
|
|
| 202 |
detectedValues[n++].type = DataTypes.INT;
|
|
| 152 | 203 |
continue; |
| 153 | 204 |
} |
| 154 | 205 |
if (possibleDataType.possibleLong) {
|
| 155 |
types[n++] = DataTypes.LONG;
|
|
| 206 |
detectedValues[n++].type = DataTypes.LONG;
|
|
| 156 | 207 |
continue; |
| 157 | 208 |
} |
| 158 | 209 |
if (possibleDataType.possibleDecimal) {
|
| 159 | 210 |
// Preferimos un Decimal que un Float/Double |
| 160 |
types[n++] = DataTypes.DECIMAL;
|
|
| 211 |
detectedValues[n++].type = DataTypes.DECIMAL;
|
|
| 161 | 212 |
continue; |
| 162 | 213 |
} |
| 163 | 214 |
if (possibleDataType.possibleFloat) {
|
| 164 | 215 |
// Forzamos los float a double para evitar perder precision |
| 165 |
types[n++] = DataTypes.DOUBLE;
|
|
| 216 |
detectedValues[n++].type = DataTypes.DOUBLE;
|
|
| 166 | 217 |
continue; |
| 167 | 218 |
} |
| 168 | 219 |
if (possibleDataType.possibleDouble) {
|
| 169 |
types[n++] = DataTypes.DOUBLE;
|
|
| 220 |
detectedValues[n++].type = DataTypes.DOUBLE;
|
|
| 170 | 221 |
continue; |
| 171 | 222 |
} |
| 172 | 223 |
if (possibleDataType.possibleURL) {
|
| 173 |
types[n++] = DataTypes.URL;
|
|
| 224 |
detectedValues[n++].type = DataTypes.URL;
|
|
| 174 | 225 |
continue; |
| 175 | 226 |
} |
| 176 | 227 |
if (possibleDataType.possibleDate) {
|
| 177 |
types[n++] = DataTypes.DATE;
|
|
| 228 |
detectedValues[n++].type = DataTypes.DATE;
|
|
| 178 | 229 |
continue; |
| 179 | 230 |
} |
| 180 | 231 |
if (possibleDataType.possibleGeometry) {
|
| 181 |
types[n++] = DataTypes.GEOMETRY;
|
|
| 232 |
detectedValues[n++].type = DataTypes.GEOMETRY;
|
|
| 182 | 233 |
continue; |
| 183 | 234 |
} |
| 184 |
types[n++] = DataTypes.STRING;
|
|
| 235 |
detectedValues[n++].type = DataTypes.STRING;
|
|
| 185 | 236 |
} |
| 186 | 237 |
} catch (Exception ex) {
|
| 187 | 238 |
throw new RuntimeException("Problems reading file '" + this.getFullFileName() + "' near line " + lineno + ".", ex);
|
| 188 | 239 |
} |
| 189 |
return types;
|
|
| 240 |
return detectedValues;
|
|
| 190 | 241 |
} |
| 191 | 242 |
|
| 192 | 243 |
@SuppressWarnings("UseSpecificCatch")
|
| trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVStoreProvider.java | ||
|---|---|---|
| 74 | 74 |
import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorer; |
| 75 | 75 |
import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorerParameters; |
| 76 | 76 |
import org.gvsig.fmap.dal.spi.DataStoreProviderServices; |
| 77 |
import org.gvsig.fmap.dal.store.csv.AutomaticDetectionOfTypes.DetectedValue; |
|
| 77 | 78 |
import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReader; |
| 78 | 79 |
import org.gvsig.fmap.dal.store.csv.simplereaders.FixedLenReader; |
| 79 | 80 |
import org.gvsig.fmap.dal.store.csv.simplereaders.SimpleReader; |
| ... | ... | |
| 464 | 465 |
public int geomSubtype = Geometry.SUBTYPES.GEOM2D; |
| 465 | 466 |
public Map<String,String> tags = new HashMap<>(); |
| 466 | 467 |
public Map<String,String> assignments = new HashMap<>(); |
| 468 |
|
|
| 469 |
// Valores obtenidos desde la deteccion automatica desde los datos |
|
| 470 |
public DetectedValue detectedValue = null ; |
|
| 467 | 471 |
|
| 468 | 472 |
private String typename = "string"; |
| 469 | 473 |
|
| ... | ... | |
| 593 | 597 |
|
| 594 | 598 |
} |
| 595 | 599 |
|
| 596 |
private EditableFeatureType getFeatureType(String headers[], int automaticTypes[]) {
|
|
| 600 |
private EditableFeatureType getFeatureType(String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) {
|
|
| 597 | 601 |
EditableFeatureType fType = getStoreServices().createFeatureType(this.getName()); |
| 598 | 602 |
fType.setHasOID(true); |
| 599 | 603 |
|
| ... | ... | |
| 609 | 613 |
// son los detectados automaticamente. |
| 610 | 614 |
if (automaticTypes != null) {
|
| 611 | 615 |
for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) {
|
| 612 |
fieldTypes[i].type = automaticTypes[i]; |
|
| 616 |
fieldTypes[i].detectedValue = automaticTypes[i]; |
|
| 617 |
fieldTypes[i].type = automaticTypes[i].getType(); |
|
| 613 | 618 |
} |
| 614 | 619 |
} |
| 615 | 620 |
// Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo |
| ... | ... | |
| 646 | 651 |
// |
| 647 | 652 |
for (FieldTypeParser fieldType : fieldTypes) {
|
| 648 | 653 |
EditableFeatureAttributeDescriptor fad = fType.add(fieldType.name, fieldType.type); |
| 649 |
fad.setSize(fieldType.size); |
|
| 654 |
fad.setRequiredBytes(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size)); |
|
| 655 |
fad.setSize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size)); |
|
| 656 |
if( fad.getPrecision()<fieldType.detectedValue.getPrecision() ) {
|
|
| 657 |
fad.setPrecision(fieldType.detectedValue.getPrecision()); |
|
| 658 |
} |
|
| 659 |
if( fad.getScale()<fieldType.detectedValue.getScale()) {
|
|
| 660 |
fad.setScale(fieldType.detectedValue.getScale()); |
|
| 661 |
} |
|
| 650 | 662 |
if (fieldType.type == DataTypes.GEOMETRY ) {
|
| 651 | 663 |
fad.setGeometryType(fieldType.geomType, fieldType.geomSubtype); |
| 652 | 664 |
if( fType.getDefaultGeometryAttributeName() == null ) {
|
| ... | ... | |
| 693 | 705 |
String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters()); |
| 694 | 706 |
if ( pointDimensionNames != null ) {
|
| 695 | 707 |
PointAttributeEmulator emulator = new PointAttributeEmulator(pointDimensionNames); |
| 696 |
EditableFeatureAttributeDescriptor attr = fType.add("the_geom", DataTypes.GEOMETRY, emulator);
|
|
| 708 |
String columnName = CSVStoreParameters.getPointColumnName(this.getParameters()); |
|
| 709 |
if( StringUtils.isBlank(columnName) ) {
|
|
| 710 |
columnName = "the_geom"; |
|
| 711 |
} |
|
| 712 |
EditableFeatureAttributeDescriptor attr = fType.add(columnName, DataTypes.GEOMETRY, emulator); |
|
| 697 | 713 |
GeometryManager geommgr = GeometryLocator.getGeometryManager(); |
| 698 | 714 |
GeometryType gt; |
| 699 | 715 |
try {
|
| ... | ... | |
| 959 | 975 |
} |
| 960 | 976 |
} |
| 961 | 977 |
|
| 962 |
int[] detectedTypes = automaticDetectionOfTypes(headers);
|
|
| 978 |
AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = automaticDetectionOfTypes(headers);
|
|
| 963 | 979 |
if (detectedTypes != null && detectedTypes.length > headers.length) {
|
| 964 | 980 |
// Se han detectado mas columnas que las que hay en la cabezera, |
| 965 | 981 |
// a?adimos mas columnas a la cabezera. |
| ... | ... | |
| 1110 | 1126 |
} |
| 1111 | 1127 |
} |
| 1112 | 1128 |
|
| 1113 |
private int[] automaticDetectionOfTypes(String[] headers) throws IOException {
|
|
| 1129 |
private AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(String[] headers) throws IOException {
|
|
| 1114 | 1130 |
boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(getCSVParameters()); |
| 1115 | 1131 |
if (!automatic_types_detection) {
|
| 1116 | 1132 |
return null; |
| 1117 | 1133 |
} |
| 1118 |
int[] types = null;
|
|
| 1134 |
AutomaticDetectionOfTypes.DetectedValue[] types = null;
|
|
| 1119 | 1135 |
|
| 1120 | 1136 |
FileReader in = null; |
| 1121 | 1137 |
SimpleReader reader = null; |
Also available in: Unified diff