Application: gvSIG desktop

package org.gvsig.fmap.dal.store.gml;

import java.io.File;

import java.io.FileInputStream;

import java.io.InputStreamReader;

import java.io.Reader;

import java.nio.charset.Charset;

import java.nio.charset.StandardCharsets;

import java.util.ArrayList;

import java.util.Collections;

import java.util.HashMap;

import java.util.HashSet;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.LinkedHashSet;

import java.util.List;

import java.util.Map;

import java.util.Set;

import javax.xml.parsers.SAXParser;

import javax.xml.parsers.SAXParserFactory;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.lang3.mutable.MutableLong;

import org.gvsig.tools.namestranslator.NamesTranslator;

import org.xml.sax.Attributes;

import org.xml.sax.InputSource;

import org.xml.sax.Locator;

import org.xml.sax.SAXException;

import org.xml.sax.helpers.DefaultHandler;

public class TestXML {

    private static final String XMLFILE1 = "/home/jjdelcerro/datos/geodata/vector/sigpac/2018/Declaracion.xml";

    private static final String XMLFILE2 = "/home/jjdelcerro/datos/geodata/vector/ARENA2/quincenas-0/TV_03_2019_01_Q1/victimas.xml";

    private static final String XMLFILE_BIG = "/home/jjdelcerro/datos/geodata/vector/RSUPAC/2020/BDA_RSU_PAC20_1713052020_001.XML";

    public static void main(String[] args) throws Exception {

        TestXML t = new TestXML();

        List<String> tags = new ArrayList(t.extractTags(XMLFILE_BIG));

        Collections.sort(tags);

//        System.out.println("Tags: -------------------------");

//        for (String tag : tags) {

//            System.out.println(tag);

//        }

        Map<String,List<String>> tables = new LinkedHashMap<>();

        Map<String,Set<String>> tables_tmp = new LinkedHashMap<>();

        for (String tag1 : tags) {

            String tableName = tag1;

            Set fields = new HashSet<>();

            for (String tag2 : tags) {

                if( tableName.equals(tag2) ) {

                    continue;

                }

                if( tag2.startsWith(tableName) ) {

                    String fieldName = tag2.substring(tableName.length()+1);

                    if( !fieldName.contains("/") ) {

                        fields.add(fieldName);

                    }

                }

            }

            if( !fields.isEmpty() ) {

                tables_tmp.put(tableName,fields);

            }

        }

        Map<String,String> tableNames = new HashMap<>();

//        NamesTranslator nt = NamesTranslator.createTrimTranslator(45);

        for (Map.Entry<String, Set<String>> entry : tables_tmp.entrySet()) {

            String fullTableName = entry.getKey();

            String tableName = FilenameUtils.getBaseName(fullTableName);

//            int i = nt.addSource(fullTableName);

//            tableNames.put(tableName,nt.getTranslation(i));

            tableNames.put(fullTableName,tableName);

        }

        for (Map.Entry<String, Set<String>> entry : tables_tmp.entrySet()) {

            String tableName = entry.getKey();

            List<String> fields = new ArrayList<>();

            String[] fieldkeys = tableName.split("/");

            for (int i = 0; i < fieldkeys.length; i++) {

                String fieldkey = fieldkeys[i];

                if( i==fieldkeys.length-1 ) {

                    fields.add("$ID_"+fieldkey+", PK");

                } else {

                    fields.add("$ID_"+fieldkey+", FK");

                }

            }

            for (String fieldName : entry.getValue()) {

                if( tables_tmp.containsKey(tableName+"/"+fieldName) ) {

                    fields.add("$ID_"+fieldName+" FK");

                } else {

                    fields.add(fieldName);

                }

            }            

            Collections.sort(fields);

            tables.put(tableName, fields);

        }

        int n = 1;

        for (Map.Entry<String, List<String>> entry : tables.entrySet()) {

            String tableName = entry.getKey();

            List<String> fields = entry.getValue();

            System.out.println("TABLE "+tableNames.get(tableName)+" ("+n+", "+tableName+")");

            for (String field : fields) {

                System.out.println("    "+field);

            }

            n++;

        }

//        List<List<String>> records = t.getRecords1(

//                XMLFILE1, 

//                "DECLARACION/LINEA_DECLARACION", 

//                ListBuilder.create(

//                    "DECLARACION/LINEA_DECLARACION/PROV",

////                    "DECLARACION/LINEA_DECLARACION/MUN_INE",

////                    "DECLARACION/LINEA_DECLARACION/MUN_CAT",

////                    "DECLARACION/LINEA_DECLARACION/AGREGADO",

//                    "DECLARACION/LINEA_DECLARACION/ZONA",

//                    "DECLARACION/LINEA_DECLARACION/POLIGONO",

//                    "DECLARACION/LINEA_DECLARACION/PARCELA",

//                    "DECLARACION/LINEA_DECLARACION/RECINTO",

//                    "DECLARACION/LINEA_DECLARACION/PARCELA_AGRICOLA",

//                    "DECLARACION/LINEA_DECLARACION/CULTIVO",

////                    "DECLARACION/LINEA_DECLARACION/WKT",

//                    "DECLARACION/LINEA_DECLARACION/DN_SURFACE",

//                    "DECLARACION/LINEA_DECLARACION/SUPERFICIE_DECLARADA"

////                    "DECLARACION/LINEA_DECLARACION/FC_ALMENDROS",

////                    "DECLARACION/LINEA_DECLARACION/FC_ALGARROBOS",

////                    "DECLARACION/LINEA_DECLARACION/FC_AVELLANOS",

////                    "DECLARACION/LINEA_DECLARACION/FC_NOGALES",

////                    "DECLARACION/LINEA_DECLARACION/FC_PISTACHOS",

////                    "DECLARACION/LINEA_DECLARACION/FC_TOTAL"

//                )

//        );

//        for (List<String> record : records) {

//            System.out.println(StringUtils.join(record, ","));

//        }

    }

    private Reader openFileReader(File xmlfile) throws Exception  {

        FileInputStream fis = new FileInputStream(xmlfile);

//        EncodingDetector encodingDetector = TikaConfig.getDefaultConfig().getEncodingDetector();

//        Charset encoding = encodingDetector.detect(fis, null);

        Charset encoding = StandardCharsets.UTF_8;

        InputStreamReader reader = new InputStreamReader(fis, encoding);

        return reader;

    }

    private Set<String> extractTags(String xmlfile) throws Exception {

        SAXParserFactory spf = SAXParserFactory.newInstance();

        spf.setNamespaceAware(true);

        SAXParser saxParser = spf.newSAXParser();

//        RandomAccessFileReader reader = new RandomAccessFileReader(new File(xmlfile), StandardCharsets.UTF_8);

        Reader reader = openFileReader(new File(xmlfile));

        InputSource is = new InputSource(reader);

        List<String> path = new ArrayList<>();

        Set<String> tags = new LinkedHashSet<>();

//        System.out.println("Parse (extract-tags): -------------------------");

        saxParser.parse(is, new DefaultHandler() {

            private Locator locator;

            @Override

            public void setDocumentLocator(Locator locator) {

                this.locator = locator;

            }

            @Override

            public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

                    path.add(localName);

                    int line = this.locator.getLineNumber();

                    int column = this.locator.getColumnNumber()-2-localName.length();

                    String path_s = StringUtils.join(path, "/");

                    tags.add(path_s);

                    for (int i = 0; i < attributes.getLength(); i++) {

                        String name = attributes.getLocalName(i);

                        tags.add(path_s+"/#"+name);

                    }

//                    if( "DECLARACION/LINEA_DECLARACION".equals(StringUtils.join(path, "/")) ) {

//                        System.out.println(path_s+": "+line+":"+column);

//                    }

//                    if( "DECLARACION".equals(StringUtils.join(path, "/")) ) {

//                        System.out.println(path_s+": "+line+":"+column);

//                    }

            }

            @Override

            public void endElement(String uri, String localName, String qName) throws SAXException {

                    path.remove(path.size()-1);

            }

        });

        return tags;

    }

    private List<List<String>> getRecords1(String xmlfile, String recordPath, List<String>fieldPaths) throws Exception {

        class ParseRecordsHandler extends DefaultHandler {

            Locator locator;

            List<String> path = new ArrayList<>();

            List<List<String>> records = new ArrayList<>();

            Map<String,String> record = new HashMap<>();

            StringBuilder value = new StringBuilder();

            @Override

            public void setDocumentLocator(Locator locator) {

                this.locator = locator;

            }

            @Override

            public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

                    path.add(localName);

                    int line = this.locator.getLineNumber();

                    int column = this.locator.getColumnNumber()-2-localName.length();

                    String path_s = StringUtils.join(path, "/");

                    if( StringUtils.equalsIgnoreCase(path_s, recordPath) ) {

                        return;

                    }

                    this.value.setLength(0);

//                    this.value.trimToSize();

            }

            @Override

            public void characters(char[] ch, int start, int length) throws SAXException {

                value.append(new String(ch, start, length));

            }

            @Override

            public void endElement(String uri, String localName, String qName) throws SAXException {

                    int line = this.locator.getLineNumber();

                    int column = this.locator.getColumnNumber()-2-localName.length();

                    String path_s = StringUtils.join(path, "/");

                    if( StringUtils.equalsIgnoreCase(path_s, recordPath) ) {

                        List<String> values = new ArrayList<>();

                        for (String fieldPath : fieldPaths) {

                            values.add(record.get(fieldPath));

                        }

                        records.add(values);

                        record.clear();

                    } else {

                        for (String fieldPath : fieldPaths) {

                            if( StringUtils.equalsIgnoreCase(path_s, fieldPath) ) {

                                record.put(fieldPath, this.value.toString());

                                // No break to handle repeated columns

                            }

                        }

                    }

                    path.remove(path.size()-1);

            }

        }

        SAXParserFactory spf = SAXParserFactory.newInstance();

        spf.setNamespaceAware(true);

        SAXParser saxParser = spf.newSAXParser();

//        RandomAccessFileReader reader = new RandomAccessFileReader(new File(xmlfile), StandardCharsets.UTF_8);

        Reader reader = openFileReader(new File(xmlfile));

        InputSource is = new InputSource(reader);

        ParseRecordsHandler handler = new ParseRecordsHandler();

        System.out.println("Parse (getRecords): -------------------------");

        saxParser.parse(is, handler);

        return handler.records;

    }

    private long getRecordCount(String xmlfile, String recordPath) throws Exception {

        SAXParserFactory spf = SAXParserFactory.newInstance();

        spf.setNamespaceAware(true);

        SAXParser saxParser = spf.newSAXParser();

        Reader reader = openFileReader(new File(xmlfile));

        InputSource is = new InputSource(reader);

        List<String> path = new ArrayList<>();

        MutableLong numRecords = new MutableLong(0);

        System.out.println("Parse (countRecords)");

        saxParser.parse(is, new DefaultHandler() {

            @Override

            public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

                    path.add(localName);

                    String path_s = StringUtils.join(path, "/");

                    if( StringUtils.equalsIgnoreCase(path_s, recordPath) ) {

                        numRecords.increment();

                    }

            }

            @Override

            public void endElement(String uri, String localName, String qName) throws SAXException {

                    path.remove(path.size()-1);

            }

        });

        return numRecords.longValue();

    }

//    private void createIndex(String xmlfile, String recordPath) throws Exception {

//        

//        long countRecords = getRecordCount(xmlfile, recordPath);

//        if (countRecords < 1) {

//            return;

//        }

//        RandomAccessFileIndex record_idx = new RandomAccessFileIndex();

//        record_idx.create(fileIndex, countRecords);

//

//        SAXParserFactory spf = SAXParserFactory.newInstance();

//        spf.setNamespaceAware(true);

//        SAXParser saxParser = spf.newSAXParser();

//        Reader reader = openFileReader(new File(xmlfile));

//        InputSource is = new InputSource(reader);

//        List<String> path = new ArrayList<>();

//

//        System.out.println("Parse (createIndex)");

//        saxParser.parse(is, new DefaultHandler() {

//            Locator locator;

//            

//            @Override

//            public void setDocumentLocator(Locator locator) {

//                this.locator = locator;

//            }

//            @Override

//            public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

//                    path.add(localName);

//                    int line = this.locator.getLineNumber();

//                    int column = this.locator.getColumnNumber()-2-localName.length();

//                    String path_s = StringUtils.join(path, "/");

//

//                    if( StringUtils.equalsIgnoreCase(path_s, recordPath) ) {

//                        record_idx.set(lineno++, position);

//                    }

//            }

//

//            @Override

//            public void endElement(String uri, String localName, String qName) throws SAXException {

//                    path.remove(path.size()-1);

//            }

//        });

//

//        return numRecords.longValue();

//    }

    private void test() {

    }

}