package org.unicode.cldr.tool;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.unicode.cldr.tool.Option.Options;
import org.unicode.cldr.util.Builder;
import org.unicode.cldr.util.CLDRConfig;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRPaths;
import org.unicode.cldr.util.CldrUtility;
import org.unicode.cldr.util.Counter;
import org.unicode.cldr.util.DtdData;
import org.unicode.cldr.util.DtdData.Attribute;
import org.unicode.cldr.util.DtdData.Element;
import org.unicode.cldr.util.DtdType;
import org.unicode.cldr.util.PathStarrer;
import org.unicode.cldr.util.PatternCache;
import org.unicode.cldr.util.RegexUtilities;
import org.unicode.cldr.util.SupplementalDataInfo;
import org.unicode.cldr.util.XMLFileReader;
import org.unicode.cldr.util.XMLFileReader.SimpleHandler;
import org.unicode.cldr.util.XPathParts;
import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;

import com.google.common.base.Splitter;
import com.ibm.icu.dev.util.BagFormatter;
import com.ibm.icu.dev.util.CollectionUtilities;
import com.ibm.icu.impl.Relation;
import com.ibm.icu.impl.Row;
import com.ibm.icu.impl.Row.R2;
import com.ibm.icu.impl.Row.R4;
import com.ibm.icu.util.VersionInfo;

public class GenerateItemCounts {
    private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = CLDRConfig.getInstance().getSupplementalDataInfo();
    private static final boolean SKIP_ORDERING = true;
    private static final String OUT_DIRECTORY = CLDRPaths.GEN_DIRECTORY + "/itemcount/"; // CldrUtility.MAIN_DIRECTORY;
    private Map<String, List<StackTraceElement>> cantRead = new TreeMap<String, List<StackTraceElement>>();

    private static String[] DIRECTORIES = {
        // MUST be oldest first!
        // "cldr-archive/cldr-21.0",
        // "cldr-24.0",
        "cldr-27.0",
        "trunk"
    };

    private static String TRUNK_VERSION = "26.0";

    static boolean doChanges = true;
    static Relation<String, String> path2value = Relation.of(new TreeMap<String, Set<String>>(), TreeSet.class);
    static final AttributeTypes ATTRIBUTE_TYPES = new AttributeTypes();

    final static Options myOptions = new Options();

    enum MyOptions {
        summary(null, null, "if present, summarizes data already collected. Run once with, once without."),
        directory(".*", ".*",
            "if summary, creates filtered version (eg -d main): does a find in the name, which is of the form dir/file"),
            verbose(null, null, "verbose debugging messages"),
            rawfilter(".*", ".*", "filter the raw files (non-summary, mostly for debugging)"), ;
        // boilerplate
        final Option option;

        MyOptions(String argumentPattern, String defaultArgument, String helpText) {
            option = myOptions.add(this, argumentPattern, defaultArgument, helpText);
        }
    }

    static Matcher DIR_FILE_MATCHER;
    static Matcher RAW_FILE_MATCHER;
    static boolean VERBOSE;

    public static void main(String[] args) throws IOException {
        myOptions.parse(MyOptions.directory, args, true);

        DIR_FILE_MATCHER = PatternCache.get(MyOptions.directory.option.getValue()).matcher("");
        RAW_FILE_MATCHER = PatternCache.get(MyOptions.rawfilter.option.getValue()).matcher("");
        VERBOSE = MyOptions.verbose.option.doesOccur();

        if (MyOptions.summary.option.doesOccur()) {
            doSummary();
            System.out.println("DONE");
            return;
            // } else if (arg.equals("changes")) {
            // doChanges = true;
        } else {
        }
        // Pattern dirPattern = dirPattern = PatternCache.get(arg);
        GenerateItemCounts main = new GenerateItemCounts();
        try {
            Relation<String, String> oldPath2value = null;
            for (String dir : DIRECTORIES) {
                // if (dirPattern != null && !dirPattern.matcher(dir).find()) continue;
                final String pathname = dir.equals("trunk") ? CLDRPaths.BASE_DIRECTORY
                    : CLDRPaths.ARCHIVE_DIRECTORY + "/" + dir;
                boolean isFinal = dir == DIRECTORIES[DIRECTORIES.length - 1];

                String fulldir = new File(pathname).getCanonicalPath();
                String prefix = (MyOptions.rawfilter.option.doesOccur() ? "filtered_" : "");
                String fileKey = dir.replace("/", "_");
                try (
                    PrintWriter summary = BagFormatter.openUTF8Writer(OUT_DIRECTORY, prefix + fileKey + "_count.txt");
                    PrintWriter changes = BagFormatter
                        .openUTF8Writer(OUT_DIRECTORY, prefix + fileKey + "_changes.txt");
                    PrintWriter changesNew = BagFormatter
                        .openUTF8Writer(OUT_DIRECTORY, prefix + fileKey + "_news.txt");
                    PrintWriter changesDeletes = BagFormatter
                        .openUTF8Writer(OUT_DIRECTORY, prefix + fileKey + "_deletes.txt");
                    PrintWriter changesSummary = BagFormatter
                        .openUTF8Writer(OUT_DIRECTORY, prefix + fileKey + "_changes_summary.txt");) {
                    main.summarizeCoverage(summary, fulldir, isFinal);
                    if (doChanges) {
                        if (oldPath2value != null) {
                            compare(summary, changes, changesNew, changesDeletes, changesSummary, oldPath2value, path2value);
                            checkBadAttributes(path2value, prefix + fileKey + "_dtd_check.txt");
                        }
                        oldPath2value = path2value;
                        path2value = Relation.of(new TreeMap<String, Set<String>>(), TreeSet.class);
                    }
                }
            }
            ATTRIBUTE_TYPES.showStarred();
        } finally {
            if (main.cantRead.size() != 0) {
                System.out.println("Couldn't read:\t");
                for (String file : main.cantRead.keySet()) {
                    System.out.println(file + "\t" + main.cantRead.get(file));
                }
            }
            System.out.println("DONE");
        }
    }

    static final Set<String> SKIP_ATTRIBUTES = new HashSet<>(Arrays.asList("draft", "references", "validSubLocales"));

    static final Relation<String, DtdType> ELEMENTS_OCCURRING = Relation.of(new TreeMap(), TreeSet.class);
    static final Relation<String, DtdType> ELEMENTS_POSSIBLE = Relation.of(new TreeMap(), TreeSet.class);
    static final Relation<String, Row.R2<DtdType, String>> ATTRIBUTES_OCCURRING = Relation.of(new TreeMap(), TreeSet.class);
    static final Relation<String, Row.R2<DtdType, String>> ATTRIBUTES_POSSIBLE = Relation.of(new TreeMap(), TreeSet.class);

    private static void checkBadAttributes(Relation<String, String> path2value2, String outputFile)
        throws IOException {
        // an attribute is misplaced if it is not distinguishing, but is on a non-final node.

        Set<String> errors = new LinkedHashSet<>();

        SupplementalDataInfo supp = SUPPLEMENTAL_DATA_INFO;
        for (DtdType dtdType : DtdType.values()) {
            if (dtdType == DtdType.ldmlICU) {
                continue;
            }
            DtdData data = DtdData.getInstance(dtdType);
            for (Element element : data.getElements()) {
                String elementName = element.name;
                ELEMENTS_POSSIBLE.put(elementName, dtdType);
                final Set<Element> children = element.getChildren().keySet();

                boolean skipFinal = children.isEmpty()
                    || children.size() == 1
                    && children.iterator().next().name.equals("special");

                for (Entry<Attribute, Integer> attributeInt : element.getAttributes().entrySet()) {
                    Attribute attribute = attributeInt.getKey();
                    String attributeName = attribute.name;
                    if (attribute.defaultValue != null) {
                        errors.add("Warning, default value «" + attribute.defaultValue
                            + "» for: " + dtdType + "\t" + elementName + "\t" + attributeName);
                    }
                    final R2<DtdType, String> attributeRow = Row.of(dtdType, elementName);
                    ATTRIBUTES_POSSIBLE.put(attributeName, attributeRow);
                    if (skipFinal || SKIP_ATTRIBUTES.contains(attributeName)) { // don't worry about non-final, references, draft, standard
                        continue;
                    }
                    if (supp.isDeprecated(dtdType, elementName, attributeName, null)) {
                        continue;
                    }
                    if (!CLDRFile.isDistinguishing(dtdType, elementName, attributeName)) {
                        String doesOccur = "";
                        final Set<R2<DtdType, String>> attributeRows = ATTRIBUTES_OCCURRING.get(attributeName);
                        if (attributeRows == null || !attributeRows.contains(attributeRow)) {
                            doesOccur = "\tNEVER";
                        }
                        errors.add("Warning, !disting, !leaf: " + dtdType + "\t" + elementName + "\t" + attributeName + "\t" + children + doesOccur);
                    }
                }
            }
        }
        try (
            PrintWriter out = BagFormatter.openUTF8Writer(OUT_DIRECTORY, outputFile)) {
            out.println("\nElements\tDeprecated\tOccurring\tPossible in DTD, but never occurs");

            for (Entry<String, Set<DtdType>> x : ELEMENTS_POSSIBLE.keyValuesSet()) {
                final String element = x.getKey();
                if (element.equals("#PCDATA") || element.equals("ANY") || element.equals("generation")) {
                    continue;
                }
                final Set<DtdType> possible = x.getValue();
                Set<DtdType> deprecated = new TreeSet();
                for (DtdType dtdType : possible) {
                    if (SUPPLEMENTAL_DATA_INFO.isDeprecated(dtdType, element, "*", "*")) {
                        deprecated.add(dtdType);
                    }
                }
                Set<DtdType> notDeprecated = new TreeSet(possible);
                notDeprecated.removeAll(deprecated);

                Set<DtdType> occurs = CldrUtility.ifNull(ELEMENTS_OCCURRING.get(element), Collections.EMPTY_SET);
                Set<DtdType> noOccur = new TreeSet(possible);
                noOccur.removeAll(occurs);

                if (!Collections.disjoint(deprecated, occurs)) { // deprecated must not occur
                    final Set<DtdType> intersection = CldrUtility.intersect(deprecated, occurs);
                    errors.add("Error: element «" + element
                        + "» is deprecated in " + (deprecated.equals(possible) ? "EVERYWHERE" : intersection) +
                        " but occurs in live data: " + intersection);
                }
                if (!Collections.disjoint(notDeprecated, noOccur)) { // if !deprecated & !occur, warning
                    errors.add("Warning: element «" + element
                        + "» doesn't occur in and is not deprecated in " + CldrUtility.intersect(notDeprecated, noOccur));
                }

                out.println(element
                    + "\t" + deprecated
                    + "\t" + occurs
                    + "\t" + noOccur
                    );
            }

            out.println("\nAttributes\tDeprecated\tOccurring\tPossible in DTD, but never occurs");

            for (Entry<String, Set<R2<DtdType, String>>> x : ATTRIBUTES_POSSIBLE.keyValuesSet()) {
                final String attribute = x.getKey();
                if (attribute.equals("alt") || attribute.equals("draft") || attribute.equals("references")) {
                    continue;
                }
                final Set<R2<DtdType, String>> possible = x.getValue();
                Set<R2<DtdType, String>> deprecated = new TreeSet();
                for (R2<DtdType, String> s : possible) {
                    final DtdType dtdType = s.get0();
                    final String element = s.get1();
                    if (SUPPLEMENTAL_DATA_INFO.isDeprecated(dtdType, element, attribute, "*")) {
                        deprecated.add(s);
                    }
                }
                Set<R2<DtdType, String>> notDeprecated = new TreeSet(possible);
                notDeprecated.removeAll(deprecated);

                Set<R2<DtdType, String>> occurs = CldrUtility.ifNull(ATTRIBUTES_OCCURRING.get(attribute), Collections.EMPTY_SET);
                Set<R2<DtdType, String>> noOccur = new TreeSet(possible);
                noOccur.removeAll(occurs);

                if (!Collections.disjoint(deprecated, occurs)) { // deprecated must not occur
                    final Set<R2<DtdType, String>> intersection = CldrUtility.intersect(deprecated, occurs);
                    errors.add("Error: attribute «" + attribute
                        + "» is deprecated in " + (deprecated.equals(possible) ? "EVERYWHERE" : intersection) +
                        " but occurs in live data: " + intersection);
                }
                if (!Collections.disjoint(notDeprecated, noOccur)) { // if !deprecated & !occur, warning
                    errors.add("Warning: attribute «" + attribute
                        + "» doesn't occur in and is not deprecated in " + CldrUtility.intersect(notDeprecated, noOccur));
                }
                out.println(attribute
                    + "\t" + deprecated
                    + "\t" + occurs
                    + "\t" + noOccur
                    );
            }
            out.println("\nERRORS/WARNINGS");
            out.println(CollectionUtilities.join(errors, "\n"));
        }
    }

    static class AttributeTypes {
        Relation<String, String> elementPathToAttributes = Relation.of(new TreeMap<String, Set<String>>(),
            TreeSet.class);
        final PathStarrer PATH_STARRER = new PathStarrer().setSubstitutionPattern("*");
        final Set<String> STARRED_PATHS = new TreeSet<String>();
        XPathParts parts = new XPathParts();
        StringBuilder elementPath = new StringBuilder();

        public void add(String path) {
            parts.set(path);
            elementPath.setLength(0);
            //DtdType type = CLDRFile.DtdType.valueOf(parts.getElement(0));
            for (int i = 0; i < parts.size(); ++i) {
                String element = parts.getElement(i);
                elementPath.append('/').append(element);
                elementPathToAttributes.putAll(elementPath.toString().intern(), parts.getAttributeKeys(i));
            }
        }

        public void showStarred() throws IOException {
            PrintWriter starred = BagFormatter.openUTF8Writer(OUT_DIRECTORY, "starred" + ".txt");

            for (Entry<String, Set<String>> entry : elementPathToAttributes.keyValuesSet()) {
                Set<String> attributes = entry.getValue();
                if (attributes.size() == 0) {
                    continue;
                }
                String path = entry.getKey();
                String[] elements = path.split("/");
                DtdType type = DtdType.valueOf(elements[1]);
                String finalElement = elements[elements.length - 1];
                starred.print(path);
                for (String attribute : attributes) {
                    if (CLDRFile.isDistinguishing(type, finalElement, attribute)) {
                        starred.print("[@" + attribute + "='disting.']");
                    } else {
                        starred.print("[@" + attribute + "='DATA']");
                    }
                }
                starred.println();
            }
            starred.close();
        }
    }

    static Pattern prefix = PatternCache.get("([^/]+/[^/]+)(.*)");

    static class Delta {
        Counter<String> newCount = new Counter<String>();
        Counter<String> deletedCount = new Counter<String>();
        Counter<String> changedCount = new Counter<String>();
        Counter<String> unchangedCount = new Counter<String>();

        void print(PrintWriter changesSummary, Set<String> prefixes) {
            changesSummary.println("Total"
                + "\t" + unchangedCount.getTotal()
                + "\t" + deletedCount.getTotal()
                + "\t" + changedCount.getTotal()
                + "\t" + newCount.getTotal()
                );
            changesSummary.println("Directory\tSame\tRemoved\tChanged\tAdded");
            for (String prefix : prefixes) {
                changesSummary.println(prefix
                    + "\t" + unchangedCount.get(prefix)
                    + "\t" + deletedCount.get(prefix)
                    + "\t" + changedCount.get(prefix)
                    + "\t" + newCount.get(prefix)
                    );
            }
        }
    }

    private static void compare(PrintWriter summary, PrintWriter changes, PrintWriter changesNew,
        PrintWriter changesDeletes, PrintWriter changesSummary, Relation<String, String> oldPath2value,
        Relation<String, String> path2value2) {
        Set<String> union = Builder.with(new TreeSet<String>()).addAll(oldPath2value.keySet())
            .addAll(path2value2.keySet()).get();
        long total = 0;
        Matcher prefixMatcher = prefix.matcher("");
        Delta charCount = new Delta();
        Delta itemCount = new Delta();
        Counter<String> newLength = new Counter<String>();
        Counter<String> deletedLength = new Counter<String>();
        Counter<String> changedLength = new Counter<String>();
        Counter<String> unchangedLength = new Counter<String>();
        Set<String> prefixes = new TreeSet();
        for (String path : union) {
            if (!prefixMatcher.reset(path).find()) {
                throw new IllegalArgumentException();
            }
            String prefix = prefixMatcher.group(1);
            prefixes.add(prefix);
            String localPath = prefixMatcher.group(2);
            Set<String> set1 = oldPath2value.getAll(path);
            Set<String> set2 = path2value2.getAll(path);
            if (set2 != null) {
                total += set2.size();
            }
            if (set1 == null) {
                changesNew.println(prefix + "\t" + "\t" + set2 + "\t" + localPath);
                itemCount.newCount.add(prefix, set2.size());
                charCount.newCount.add(prefix, totalLength(set2));
            } else if (set2 == null) {
                changesDeletes.println(prefix + "\t" + set1 + "\t\t" + localPath);
                itemCount.deletedCount.add(prefix, -set1.size());
                charCount.deletedCount.add(prefix, -totalLength(set1));
            } else if (!set1.equals(set2)) {
                TreeSet<String> set1minus2 = Builder.with(new TreeSet<String>()).addAll(set1).removeAll(set2).get();
                TreeSet<String> set2minus1 = Builder.with(new TreeSet<String>()).addAll(set2).removeAll(set1).get();
                TreeSet<String> set2and1 = Builder.with(new TreeSet<String>()).addAll(set2).retainAll(set1).get();
                itemCount.changedCount.add(prefix, (set2minus1.size() + set1minus2.size() + 1) / 2);
                itemCount.unchangedCount.add(prefix, set2and1.size());
                charCount.changedCount.add(prefix, (totalLength(set2minus1) + totalLength(set1minus2) + 1) / 2);
                charCount.unchangedCount.add(prefix, totalLength(set2and1));
                changes.println(prefix + "\t" + set1minus2
                    + "\t"
                    + set2minus1
                    + "\t" + localPath);
            } else {
                itemCount.unchangedCount.add(prefix, set2.size());
                charCount.unchangedCount.add(prefix, totalLength(set2));
            }
        }
        itemCount.print(changesSummary, prefixes);
        changesSummary.println();
        charCount.print(changesSummary, prefixes);
//        union = Builder.with(new TreeSet<String>())
//            .addAll(newCount.keySet())
//            .addAll(deletedCount.keySet())
//            .addAll(changedCount.keySet())
//            .addAll(unchangedCount.keySet())
//            .get();
        summary.println("#Total:\t" + total);
    }

    private static long totalLength(Set<String> set2) {
        int result = 0;
        for (String s : set2) {
            result += s.length();
        }
        return result;
    }

    final static Pattern LOCALE_PATTERN = PatternCache.get(
        "([a-z]{2,3})(?:[_-]([A-Z][a-z]{3}))?(?:[_-]([a-zA-Z0-9]{2,3}))?([_-][a-zA-Z0-9]{1,8})*");

    public static void doSummary() throws IOException {
        Map<String, R4<Counter<String>, Counter<String>, Counter<String>, Counter<String>>> key_release_count = new TreeMap<String, R4<Counter<String>, Counter<String>, Counter<String>, Counter<String>>>();
        Matcher countryLocale = LOCALE_PATTERN.matcher("");
        List<String> releases = new ArrayList<String>();
        Pattern releaseNumber = PatternCache.get("count_(?:.*-(\\d+(\\.\\d+)*)|trunk)\\.txt");
        // int releaseCount = 1;
        Relation<String, String> release_keys = Relation.of(new TreeMap<String, Set<String>>(), TreeSet.class);
        Relation<String, String> localesToPaths = Relation.of(new TreeMap<String, Set<String>>(), TreeSet.class);
        Set<String> writtenLanguages = new TreeSet<String>();
        Set<String> countries = new TreeSet<String>();

        File[] listFiles = new File(OUT_DIRECTORY).listFiles();
        // find the most recent version
        VersionInfo mostRecentVersion = VersionInfo.getInstance(0);
        for (File subdir : listFiles) {
            final String name = subdir.getName();
            final Matcher releaseMatcher = releaseNumber.matcher(name);
            if (!releaseMatcher.matches()) {
                if (name.startsWith("count_")) {
                    throw new IllegalArgumentException("Bad match " + RegexUtilities.showMismatch(releaseMatcher, name));
                }
                continue;
            }
            String releaseNum = releaseMatcher.group(1); // "1." + releaseCount++;
            if (releaseNum == null) {
                releaseNum = TRUNK_VERSION;
            }
            VersionInfo vi = VersionInfo.getInstance(releaseNum);
            if (vi.compareTo(mostRecentVersion) > 0) {
                mostRecentVersion = vi;
            }
        }

        for (File subdir : listFiles) {
            final String name = subdir.getName();
            final Matcher releaseMatcher = releaseNumber.matcher(name);
            if (!releaseMatcher.matches()) {
                if (name.startsWith("count_")) {
                    throw new IllegalArgumentException("Bad match " + RegexUtilities.showMismatch(releaseMatcher, name));
                }
                continue;
            }
            String releaseNum = releaseMatcher.group(1); // "1." + releaseCount++;
            if (releaseNum == null) {
                releaseNum = TRUNK_VERSION;
            }
            VersionInfo vi = VersionInfo.getInstance(releaseNum);
            boolean captureData = vi.equals(mostRecentVersion);
            releases.add(releaseNum);
            BufferedReader in = BagFormatter.openUTF8Reader("", subdir.getCanonicalPath());
            while (true) {
                String line = in.readLine();
                if (line == null) break;
                line = line.trim();
                if (line.startsWith("#")) {
                    continue;
                }
                // common/main  New:        [Yellowknife]   /gl//ldml/dates/timeZoneNames/zone[@type="America/Yellowknife"]/exemplarCity

                String[] parts = line.split("\t");
                try {
                    String file = parts[0];
                    if (file.startsWith("seed/") || !DIR_FILE_MATCHER.reset(file).find()) {
                        if (VERBOSE) {
                            System.out.println("Skipping: " + RegexUtilities.showMismatch(DIR_FILE_MATCHER, file));
                        }
                        continue;
                    } else if (VERBOSE) {
                        System.out.println("Including: " + file);
                    }

                    long valueCount = Long.parseLong(parts[1]);
                    long valueLen = Long.parseLong(parts[2]);
                    long attrCount = Long.parseLong(parts[3]);
                    long attrLen = Long.parseLong(parts[4]);
                    int lastSlash = file.lastIndexOf("/");
                    String key2 = file;
                    String path = file.substring(0, lastSlash);
                    String key = file.substring(lastSlash + 1);
                    if (countryLocale.reset(key).matches()) {
                        String lang = countryLocale.group(1);
                        String script = countryLocale.group(2);
                        String country = countryLocale.group(3);
                        String writtenLang = lang + (script == null ? "" : "_" + script);
                        String locale = writtenLang + (country == null ? "" : "_" + country);
                        if (captureData) {
                            localesToPaths.put(locale, path);
                            writtenLanguages.add(writtenLang);
                            if (country != null) {
                                countries.add(country);
                            }
                        }
                        // System.out.println(key + " => " + newKey);
                        //key = writtenLang + "—" + ULocale.getDisplayName(writtenLang, "en");
                    }
                    if (valueCount + attrCount == 0) continue;
                    release_keys.put(releaseNum, key2);
                    R4<Counter<String>, Counter<String>, Counter<String>, Counter<String>> release_count = key_release_count
                        .get(key2);
                    if (release_count == null) {
                        release_count = Row.of(new Counter<String>(), new Counter<String>(), new Counter<String>(),
                            new Counter<String>());
                        key_release_count.put(key2, release_count);
                    }
                    release_count.get0().add(releaseNum, valueCount);
                    release_count.get1().add(releaseNum, valueLen);
                    release_count.get2().add(releaseNum, attrCount);
                    release_count.get3().add(releaseNum, attrLen);
                } catch (Exception e) {
                    throw new IllegalArgumentException(line, e);
                }
            }
            in.close();
        }
        PrintWriter summary = BagFormatter.openUTF8Writer(OUT_DIRECTORY,
            (MyOptions.directory.option.doesOccur() ? "filtered-" : "") + "summary" +
            ".txt");
        for (String file : releases) {
            summary.print("\t" + file + "\tlen");
        }
        summary.println();
        for (String key : key_release_count.keySet()) {
            summary.print(key);
            R4<Counter<String>, Counter<String>, Counter<String>, Counter<String>> release_count = key_release_count
                .get(key);
            for (String release2 : releases) {
                long count = release_count.get0().get(release2) + release_count.get2().get(release2);
                long len = release_count.get1().get(release2) + release_count.get3().get(release2);
                summary.print("\t" + count + "\t" + len);
            }
            summary.println();
        }
        for (String release : release_keys.keySet()) {
            System.out.println("Release:\t" + release + "\t" + release_keys.getAll(release).size());
        }
        summary.close();
        PrintWriter summary2 = BagFormatter.openUTF8Writer(OUT_DIRECTORY,
            (MyOptions.directory.option.doesOccur() ? "filtered-" : "") + "locales" +
            ".txt");
        summary2.println("#Languages (inc. script):\t" + writtenLanguages.size());
        summary2.println("#Countries:\t" + countries.size());
        summary2.println("#Locales:\t" + localesToPaths.size());
        for (Entry<String, Set<String>> entry : localesToPaths.keyValuesSet()) {
            summary2.println(entry.getKey() + "\t" + CollectionUtilities.join(entry.getValue(), "\t"));
        }
        summary2.close();
    }

    static final Set<String> ATTRIBUTES_TO_SKIP = Builder.with(new HashSet<String>())
        .addAll("version", "references", "standard", "draft").freeze();
    static final Pattern skipPath = PatternCache.get("" +
        "\\[\\@alt=\"[^\"]*proposed" +
        "|^//" +
        "(ldml(\\[[^/]*)?/identity" +
        "|(ldmlBCP47|supplementalData|keyboard)(\\[[^/]*)?/(generation|version)" +
        ")"
        );

    static void capture(DtdType type2, XPathParts parts) {
        for (int i = 0; i < parts.size(); ++i) {
            String element = parts.getElement(i);
            ELEMENTS_OCCURRING.put(element, type2);
            for (String attribute : parts.getAttributes(i).keySet()) {
                ATTRIBUTES_OCCURRING.put(attribute, Row.of(type2, element));
            }
        }
    }

    static class MyHandler extends SimpleHandler {
        XPathParts parts = new XPathParts();
        long valueCount;
        long valueLen;
        long attributeCount;
        long attributeLen;
        Matcher skipPathMatcher = skipPath.matcher("");
        Splitter lines = Splitter.onPattern("\n+").omitEmptyStrings().trimResults();
        String prefix;
        int orderedCount;
        DtdType type;
        private final boolean isFinal;

        MyHandler(String prefix, boolean isFinal) {
            this.prefix = prefix;
            this.isFinal = isFinal;
        }

        @Override
        public void handlePathValue(String path, String value) {
            if (type == null) {
                parts.set(path);
                type = DtdType.valueOf(parts.getElement(0));
            }

            ATTRIBUTE_TYPES.add(path);

            if (skipPathMatcher.reset(path).find()) {
                return;
            }
            String pathKey = null;
            if (doChanges) {
                // if (path.contains("/collations")) {
                // System.out.println("whoops");
                // }
                pathKey = fixKeyPath(path);
            }
            int len = value.length();
            value = value.trim();
            if (value.isEmpty() && len > 0) {
                value = " ";
            }
            if (value.length() != 0) {
                List<String> valueLines = lines.splitToList(value);
                if (valueLines.size() == 1) {
                    valueCount++;
                    valueLen += value.length();
                    if (doChanges) {
                        path2value.put(pathKey, value);
                    }
                } else {
                    int count = 0;
                    for (String v : valueLines) {
                        valueCount++;
                        valueLen += v.length();
                        if (doChanges) {
                            path2value.put(pathKey + "/_q" + count++, v);
                        }
                    }
                }
            }
            parts.set(path);
            if (isFinal) {
                capture(type, parts);
            }
            if (path.contains("[@")) {
                int i = parts.size() - 1; // only look at last item
                Collection<String> attributes = parts.getAttributeKeys(i);
                if (attributes.size() != 0) {
                    String element = parts.getElement(i);
                    for (String attribute : attributes) {
                        if (ATTRIBUTES_TO_SKIP.contains(attribute)
                            || CLDRFile.isDistinguishing(type, element, attribute)) {
                            continue;
                        }
                        String valuePart = parts.getAttributeValue(i, attribute);
                        // String[] valueParts = attrValue.split("\\s");
                        // for (String valuePart : valueParts) {
                        attributeCount++;
                        attributeLen += valuePart.length();
                        if (doChanges) {
                            path2value.put(pathKey + "/_" + attribute, valuePart);
                            // }
                        }
                    }
                }
            }
        }

        private String fixKeyPath(String path) {
            parts.set(path);
            for (int i = 0; i < parts.size(); ++i) {
                String element = parts.getElement(i);
                if (!SKIP_ORDERING) {
                    if (CLDRFile.isOrdered(element, type)) {
                        parts.addAttribute("_q", String.valueOf(orderedCount++));
                    }
                }
            }
            return prefix + CLDRFile.getDistinguishingXPath(parts.toString(), null, false);
        }
    }

    private MyHandler check(String systemID, String name, boolean isFinal) {
        MyHandler myHandler = new MyHandler(name, isFinal);
        try {
            XMLFileReader reader = new XMLFileReader().setHandler(myHandler);
            reader.read(systemID, XMLFileReader.CONTENT_HANDLER, true);
        } catch (Exception e) {
            cantRead.put(name, Arrays.asList(e.getStackTrace()));
        }
        return myHandler;

        // try {
        // FileInputStream fis = new FileInputStream(systemID);
        // XMLFileReader xmlReader = XMLFileReader.createXMLReader(true);
        // xmlReader.setErrorHandler(new MyErrorHandler());
        // MyHandler myHandler = new MyHandler();
        // smlReader
        // xmlReader.setHandler(myHandler);
        // InputSource is = new InputSource(fis);
        // is.setSystemId(systemID.toString());
        // xmlReader.parse(is);
        // fis.close();
        // return myHandler;
        // } catch (SAXParseException e) {
        // System.out.println("\t" + "Can't read " + systemID);
        // System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
        // } catch (SAXException e) {
        // System.out.println("\t" + "Can't read " + systemID);
        // System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
        // } catch (IOException e) {
        // System.out.println("\t" + "Can't read " + systemID);
        // System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
        // }
    }

    static class MyErrorHandler implements ErrorHandler {
        public void error(SAXParseException exception) throws SAXException {
            System.out.println("\nerror: " + XMLFileReader.showSAX(exception));
            throw exception;
        }

        public void fatalError(SAXParseException exception) throws SAXException {
            System.out.println("\nfatalError: " + XMLFileReader.showSAX(exception));
            throw exception;
        }

        public void warning(SAXParseException exception) throws SAXException {
            System.out.println("\nwarning: " + XMLFileReader.showSAX(exception));
            throw exception;
        }
    }

    private void summarizeCoverage(PrintWriter summary, String commonDir, boolean isFinal) {
        System.out.println(commonDir);
        summary.println("#name" + "\t" + "value-count" + "\t" + "value-len" + "\t" + "attr-count" + "\t" + "attr-len");
        File commonDirectory = new File(commonDir);
        if (!commonDirectory.exists()) {
            System.out.println("Doesn't exist:\t" + commonDirectory);
        }
        summarizeFiles(summary, commonDirectory, isFinal, 1);
    }

    static final Set<String> SKIP_DIRS = new HashSet<>(Arrays.asList("specs", "tools", "seed", "exemplars"));

    public void summarizeFiles(PrintWriter summary, File directory, boolean isFinal, int level) {
        System.out.println("\t\t\t\t\t\t\t".substring(0, level) + directory);
        int count = 0;
        for (File file : directory.listFiles()) {
            String filename = file.getName();
            if (filename.startsWith(".")) {
                // do nothing
            } else if (file.isDirectory()) {
                if (!SKIP_DIRS.contains(filename)) {
                    summarizeFiles(summary, file, isFinal, level + 1);
                }
            } else if (!filename.startsWith("#") && filename.endsWith(".xml")) {
                String name = new File(directory.getParent()).getName() + "/" + directory.getName() + "/"
                    + file.getName();
                name = name.substring(0, name.length() - 4); // strip .xml
                if (!RAW_FILE_MATCHER.reset(name).find()) {
                    continue;
                }
                if (VERBOSE) {
                    System.out.println(name);
                } else {
                    System.out.print(".");
                    if (++count > 100) {
                        count = 0;
                        System.out.println();
                    }
                    System.out.flush();
                }
                MyHandler handler = check(file.toString(), name, isFinal);
                summary.println(name + "\t" + handler.valueCount + "\t" + handler.valueLen + "\t"
                    + handler.attributeCount + "\t" + handler.attributeLen);
            }
        }
        System.out.println();
    }
}
