    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017package org.apache.commons.lang3.text;
    018
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.Collections;
    022import java.util.List;
    023import java.util.ListIterator;
    024import java.util.NoSuchElementException;
    025import java.util.StringTokenizer;
    026
    027import org.apache.commons.lang3.ArrayUtils;
    028import org.apache.commons.lang3.StringUtils;
    029
    030/**
    031 * Tokenizes a string based on delimiters (separators)
    032 * and supporting quoting and ignored character concepts.
    033 * <p>
    034 * This class can split a String into many smaller strings. It aims
    035 * to do a similar job to {@link java.util.StringTokenizer StringTokenizer},
    036 * however it offers much more control and flexibility including implementing
    037 * the {@link ListIterator} interface. By default, it is set up
    038 * like {@link StringTokenizer}.
    039 * </p>
    040 * <p>
    041 * The input String is split into a number of <em>tokens</em>.
    042 * Each token is separated from the next String by a <em>delimiter</em>.
    043 * One or more delimiter characters must be specified.
    044 * </p>
    045 * <p>
    046 * Each token may be surrounded by quotes.
    047 * The <em>quote</em> matcher specifies the quote character(s).
    048 * A quote may be escaped within a quoted section by duplicating itself.
    049 * </p>
    050 * <p>
    051 * Between each token and the delimiter are potentially characters that need trimming.
    052 * The <em>trimmer</em> matcher specifies these characters.
    053 * One usage might be to trim whitespace characters.
    054 * </p>
    055 * <p>
    056 * At any point outside the quotes there might potentially be invalid characters.
    057 * The <em>ignored</em> matcher specifies these characters to be removed.
    058 * One usage might be to remove new line characters.
    059 * </p>
    060 * <p>
    061 * Empty tokens may be removed or returned as null.
    062 * </p>
    063 * <pre>
    064 * "a,b,c"         - Three tokens "a","b","c"   (comma delimiter)
    065 * " a, b , c "    - Three tokens "a","b","c"   (default CSV processing trims whitespace)
    066 * "a, ", b ,", c" - Three tokens "a, " , " b ", ", c" (quoted text untouched)
    067 * </pre>
    068 *
    069 * <table>
    070 *  <caption>StrTokenizer properties and options</caption>
    071 *  <tr>
    072 *   <th>Property</th><th>Type</th><th>Default</th>
    073 *  </tr>
    074 *  <tr>
    075 *   <td>delim</td><td>CharSetMatcher</td><td>{ \t\n\r\f}</td>
    076 *  </tr>
    077 *  <tr>
    078 *   <td>quote</td><td>NoneMatcher</td><td>{}</td>
    079 *  </tr>
    080 *  <tr>
    081 *   <td>ignore</td><td>NoneMatcher</td><td>{}</td>
    082 *  </tr>
    083 *  <tr>
    084 *   <td>emptyTokenAsNull</td><td>boolean</td><td>false</td>
    085 *  </tr>
    086 *  <tr>
    087 *   <td>ignoreEmptyTokens</td><td>boolean</td><td>true</td>
    088 *  </tr>
    089 * </table>
    090 *
    091 * @since 2.2
    092 * @deprecated As of 3.6, use Apache Commons Text
    093 * <a href="https://commons.apache.org/proper/commons-text/javadocs/api-release/org/apache/commons/text/StringTokenizer.html">
    094 * StringTokenizer</a> instead
    095 */
    096@Deprecated
    097public class StrTokenizer implements ListIterator<String>, Cloneable {
    098
    099    private static final StrTokenizer CSV_TOKENIZER_PROTOTYPE;
    100    private static final StrTokenizer TSV_TOKENIZER_PROTOTYPE;
    101    static {
    102        CSV_TOKENIZER_PROTOTYPE = new StrTokenizer();
    103        CSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.commaMatcher());
    104        CSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
    105        CSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher());
    106        CSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher());
    107        CSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false);
    108        CSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false);
    109
    110        TSV_TOKENIZER_PROTOTYPE = new StrTokenizer();
    111        TSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.tabMatcher());
    112        TSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
    113        TSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher());
    114        TSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher());
    115        TSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false);
    116        TSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false);
    117    }
    118
    119    /**
    120     * Returns a clone of {@code CSV_TOKENIZER_PROTOTYPE}.
    121     *
    122     * @return a clone of {@code CSV_TOKENIZER_PROTOTYPE}.
    123     */
    124    private static StrTokenizer getCSVClone() {
    125        return (StrTokenizer) CSV_TOKENIZER_PROTOTYPE.clone();
    126    }
    127    /**
    128     * Gets a new tokenizer instance which parses Comma Separated Value strings
    129     * initializing it with the given input.  The default for CSV processing
    130     * will be trim whitespace from both ends (which can be overridden with
    131     * the setTrimmer method).
    132     * <p>
    133     * You must call a "reset" method to set the string which you want to parse.
    134     * </p>
    135     * @return a new tokenizer instance which parses Comma Separated Value strings
    136     */
    137    public static StrTokenizer getCSVInstance() {
    138        return getCSVClone();
    139    }
    140    /**
    141     * Gets a new tokenizer instance which parses Comma Separated Value strings
    142     * initializing it with the given input.  The default for CSV processing
    143     * will be trim whitespace from both ends (which can be overridden with
    144     * the setTrimmer method).
    145     *
    146     * @param input  the text to parse
    147     * @return a new tokenizer instance which parses Comma Separated Value strings
    148     */
    149    public static StrTokenizer getCSVInstance(final char[] input) {
    150        final StrTokenizer tok = getCSVClone();
    151        tok.reset(input);
    152        return tok;
    153    }
    154
    155    /**
    156     * Gets a new tokenizer instance which parses Comma Separated Value strings
    157     * initializing it with the given input.  The default for CSV processing
    158     * will be trim whitespace from both ends (which can be overridden with
    159     * the setTrimmer method).
    160     *
    161     * @param input  the text to parse
    162     * @return a new tokenizer instance which parses Comma Separated Value strings
    163     */
    164    public static StrTokenizer getCSVInstance(final String input) {
    165        final StrTokenizer tok = getCSVClone();
    166        tok.reset(input);
    167        return tok;
    168    }
    169    /**
    170     * Returns a clone of {@code TSV_TOKENIZER_PROTOTYPE}.
    171     *
    172     * @return a clone of {@code TSV_TOKENIZER_PROTOTYPE}.
    173     */
    174    private static StrTokenizer getTSVClone() {
    175        return (StrTokenizer) TSV_TOKENIZER_PROTOTYPE.clone();
    176    }
    177    /**
    178     * Gets a new tokenizer instance which parses Tab Separated Value strings.
    179     * The default for CSV processing will be trim whitespace from both ends
    180     * (which can be overridden with the setTrimmer method).
    181     * <p>
    182     * You must call a "reset" method to set the string which you want to parse.
    183     * </p>
    184     * @return a new tokenizer instance which parses Tab Separated Value strings.
    185     */
    186    public static StrTokenizer getTSVInstance() {
    187        return getTSVClone();
    188    }
    189    /**
    190     * Gets a new tokenizer instance which parses Tab Separated Value strings.
    191     * The default for CSV processing will be trim whitespace from both ends
    192     * (which can be overridden with the setTrimmer method).
    193     * @param input  the string to parse
    194     * @return a new tokenizer instance which parses Tab Separated Value strings.
    195     */
    196    public static StrTokenizer getTSVInstance(final char[] input) {
    197        final StrTokenizer tok = getTSVClone();
    198        tok.reset(input);
    199        return tok;
    200    }
    201
    202    /**
    203     * Gets a new tokenizer instance which parses Tab Separated Value strings.
    204     * The default for CSV processing will be trim whitespace from both ends
    205     * (which can be overridden with the setTrimmer method).
    206     * @param input  the string to parse
    207     * @return a new tokenizer instance which parses Tab Separated Value strings.
    208     */
    209    public static StrTokenizer getTSVInstance(final String input) {
    210        final StrTokenizer tok = getTSVClone();
    211        tok.reset(input);
    212        return tok;
    213    }
    214    /** The text to work on. */
    215    private char[] chars;
    216
    217    /** The parsed tokens */
    218    private String[] tokens;
    219
    220    /** The current iteration position */
    221    private int tokenPos;
    222
    223    /** The delimiter matcher */
    224    private StrMatcher delimMatcher = StrMatcher.splitMatcher();
    225
    226    /** The quote matcher */
    227    private StrMatcher quoteMatcher = StrMatcher.noneMatcher();
    228
    229    /** The ignored matcher */
    230    private StrMatcher ignoredMatcher = StrMatcher.noneMatcher();
    231
    232    /** The trimmer matcher */
    233    private StrMatcher trimmerMatcher = StrMatcher.noneMatcher();
    234
    235    /** Whether to return empty tokens as null */
    236    private boolean emptyAsNull;
    237
    238    /** Whether to ignore empty tokens */
    239    private boolean ignoreEmptyTokens = true;
    240
    241    /**
    242     * Constructs a tokenizer splitting on space, tab, newline and formfeed
    243     * as per StringTokenizer, but with no text to tokenize.
    244     * <p>
    245     * This constructor is normally used with {@link #reset(String)}.
    246     * </p>
    247     */
    248    public StrTokenizer() {
    249        this.chars = null;
    250    }
    251
    252    /**
    253     * Constructs a tokenizer splitting on space, tab, newline and formfeed
    254     * as per StringTokenizer.
    255     *
    256     * @param input  the string which is to be parsed, not cloned
    257     */
    258    public StrTokenizer(final char[] input) {
    259        this.chars = ArrayUtils.clone(input);
    260    }
    261
    262    /**
    263     * Constructs a tokenizer splitting on the specified character.
    264     *
    265     * @param input  the string which is to be parsed, not cloned
    266     * @param delim the field delimiter character
    267     */
    268    public StrTokenizer(final char[] input, final char delim) {
    269        this(input);
    270        setDelimiterChar(delim);
    271    }
    272
    273    /**
    274     * Constructs a tokenizer splitting on the specified delimiter character
    275     * and handling quotes using the specified quote character.
    276     *
    277     * @param input  the string which is to be parsed, not cloned
    278     * @param delim  the field delimiter character
    279     * @param quote  the field quoted string character
    280     */
    281    public StrTokenizer(final char[] input, final char delim, final char quote) {
    282        this(input, delim);
    283        setQuoteChar(quote);
    284    }
    285
    286    /**
    287     * Constructs a tokenizer splitting on the specified string.
    288     *
    289     * @param input  the string which is to be parsed, not cloned
    290     * @param delim the field delimiter string
    291     */
    292    public StrTokenizer(final char[] input, final String delim) {
    293        this(input);
    294        setDelimiterString(delim);
    295    }
    296
    297    /**
    298     * Constructs a tokenizer splitting using the specified delimiter matcher.
    299     *
    300     * @param input  the string which is to be parsed, not cloned
    301     * @param delim  the field delimiter matcher
    302     */
    303    public StrTokenizer(final char[] input, final StrMatcher delim) {
    304        this(input);
    305        setDelimiterMatcher(delim);
    306    }
    307
    308    /**
    309     * Constructs a tokenizer splitting using the specified delimiter matcher
    310     * and handling quotes using the specified quote matcher.
    311     *
    312     * @param input  the string which is to be parsed, not cloned
    313     * @param delim  the field delimiter character
    314     * @param quote  the field quoted string character
    315     */
    316    public StrTokenizer(final char[] input, final StrMatcher delim, final StrMatcher quote) {
    317        this(input, delim);
    318        setQuoteMatcher(quote);
    319    }
    320
    321    /**
    322     * Constructs a tokenizer splitting on space, tab, newline and formfeed
    323     * as per StringTokenizer.
    324     *
    325     * @param input  the string which is to be parsed
    326     */
    327    public StrTokenizer(final String input) {
    328        if (input != null) {
    329            chars = input.toCharArray();
    330        } else {
    331            chars = null;
    332        }
    333    }
    334
    335    /**
    336     * Constructs a tokenizer splitting on the specified delimiter character.
    337     *
    338     * @param input  the string which is to be parsed
    339     * @param delim  the field delimiter character
    340     */
    341    public StrTokenizer(final String input, final char delim) {
    342        this(input);
    343        setDelimiterChar(delim);
    344    }
    345
    346    /**
    347     * Constructs a tokenizer splitting on the specified delimiter character
    348     * and handling quotes using the specified quote character.
    349     *
    350     * @param input  the string which is to be parsed
    351     * @param delim  the field delimiter character
    352     * @param quote  the field quoted string character
    353     */
    354    public StrTokenizer(final String input, final char delim, final char quote) {
    355        this(input, delim);
    356        setQuoteChar(quote);
    357    }
    358
    359    /**
    360     * Constructs a tokenizer splitting on the specified delimiter string.
    361     *
    362     * @param input  the string which is to be parsed
    363     * @param delim  the field delimiter string
    364     */
    365    public StrTokenizer(final String input, final String delim) {
    366        this(input);
    367        setDelimiterString(delim);
    368    }
    369
    370    /**
    371     * Constructs a tokenizer splitting using the specified delimiter matcher.
    372     *
    373     * @param input  the string which is to be parsed
    374     * @param delim  the field delimiter matcher
    375     */
    376    public StrTokenizer(final String input, final StrMatcher delim) {
    377        this(input);
    378        setDelimiterMatcher(delim);
    379    }
    380
    381    /**
    382     * Constructs a tokenizer splitting using the specified delimiter matcher
    383     * and handling quotes using the specified quote matcher.
    384     *
    385     * @param input  the string which is to be parsed
    386     * @param delim  the field delimiter matcher
    387     * @param quote  the field quoted string matcher
    388     */
    389    public StrTokenizer(final String input, final StrMatcher delim, final StrMatcher quote) {
    390        this(input, delim);
    391        setQuoteMatcher(quote);
    392    }
    393
    394    /**
    395     * Unsupported ListIterator operation.
    396     * @param obj this parameter ignored.
    397     * @throws UnsupportedOperationException always
    398     */
    399    @Override
    400    public void add(final String obj) {
    401        throw new UnsupportedOperationException("add() is unsupported");
    402    }
    403
    404    /**
    405     * Adds a token to a list, paying attention to the parameters we've set.
    406     *
    407     * @param list  the list to add to
    408     * @param tok  the token to add
    409     */
    410    private void addToken(final List<String> list, String tok) {
    411        if (StringUtils.isEmpty(tok)) {
    412            if (isIgnoreEmptyTokens()) {
    413                return;
    414            }
    415            if (isEmptyTokenAsNull()) {
    416                tok = null;
    417            }
    418        }
    419        list.add(tok);
    420    }
    421
    422    /**
    423     * Checks if tokenization has been done, and if not then do it.
    424     */
    425    private void checkTokenized() {
    426        if (tokens == null) {
    427            if (chars == null) {
    428                // still call tokenize as subclass may do some work
    429                final List<String> split = tokenize(null, 0, 0);
    430                tokens = split.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
    431            } else {
    432                final List<String> split = tokenize(chars, 0, chars.length);
    433                tokens = split.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
    434            }
    435        }
    436    }
    437
    438    /**
    439     * Creates a new instance of this Tokenizer. The new instance is reset so
    440     * that it will be at the start of the token list.
    441     * If a {@link CloneNotSupportedException} is caught, return {@code null}.
    442     *
    443     * @return a new instance of this Tokenizer which has been reset.
    444     */
    445    @Override
    446    public Object clone() {
    447        try {
    448            return cloneReset();
    449        } catch (final CloneNotSupportedException ex) {
    450            return null;
    451        }
    452    }
    453
    454    /**
    455     * Creates a new instance of this Tokenizer. The new instance is reset so that
    456     * it will be at the start of the token list.
    457     *
    458     * @return a new instance of this Tokenizer which has been reset.
    459     * @throws CloneNotSupportedException if there is a problem cloning
    460     */
    461    Object cloneReset() throws CloneNotSupportedException {
    462        // this method exists to enable 100% test coverage
    463        final StrTokenizer cloned = (StrTokenizer) super.clone();
    464        if (cloned.chars != null) {
    465            cloned.chars = cloned.chars.clone();
    466        }
    467        cloned.reset();
    468        return cloned;
    469    }
    470
    471    /**
    472     * Gets the String content that the tokenizer is parsing.
    473     *
    474     * @return the string content being parsed
    475     */
    476    public String getContent() {
    477        if (chars == null) {
    478            return null;
    479        }
    480        return new String(chars);
    481    }
    482
    483    /**
    484     * Gets the field delimiter matcher.
    485     *
    486     * @return the delimiter matcher in use
    487     */
    488    public StrMatcher getDelimiterMatcher() {
    489        return this.delimMatcher;
    490    }
    491
    492    // Ignored
    493    /**
    494     * Gets the ignored character matcher.
    495     * <p>
    496     * These characters are ignored when parsing the String, unless they are
    497     * within a quoted region.
    498     * The default value is not to ignore anything.
    499     * </p>
    500     *
    501     * @return the ignored matcher in use
    502     */
    503    public StrMatcher getIgnoredMatcher() {
    504        return ignoredMatcher;
    505    }
    506
    507    /**
    508     * Gets the quote matcher currently in use.
    509     * <p>
    510     * The quote character is used to wrap data between the tokens.
    511     * This enables delimiters to be entered as data.
    512     * The default value is '"' (double quote).
    513     * </p>
    514     *
    515     * @return the quote matcher in use
    516     */
    517    public StrMatcher getQuoteMatcher() {
    518        return quoteMatcher;
    519    }
    520
    521    /**
    522     * Gets a copy of the full token list as an independent modifiable array.
    523     *
    524     * @return the tokens as a String array
    525     */
    526    public String[] getTokenArray() {
    527        checkTokenized();
    528        return tokens.clone();
    529    }
    530
    531    /**
    532     * Gets a copy of the full token list as an independent modifiable list.
    533     *
    534     * @return the tokens as a String array
    535     */
    536    public List<String> getTokenList() {
    537        checkTokenized();
    538        final List<String> list = new ArrayList<>(tokens.length);
    539        list.addAll(Arrays.asList(tokens));
    540        return list;
    541    }
    542
    543    /**
    544     * Gets the trimmer character matcher.
    545     * <p>
    546     * These characters are trimmed off on each side of the delimiter
    547     * until the token or quote is found.
    548     * The default value is not to trim anything.
    549     * </p>
    550     *
    551     * @return the trimmer matcher in use
    552     */
    553    public StrMatcher getTrimmerMatcher() {
    554        return trimmerMatcher;
    555    }
    556
    557    /**
    558     * Checks whether there are any more tokens.
    559     *
    560     * @return true if there are more tokens
    561     */
    562    @Override
    563    public boolean hasNext() {
    564        checkTokenized();
    565        return tokenPos < tokens.length;
    566    }
    567
    568    /**
    569     * Checks whether there are any previous tokens that can be iterated to.
    570     *
    571     * @return true if there are previous tokens
    572     */
    573    @Override
    574    public boolean hasPrevious() {
    575        checkTokenized();
    576        return tokenPos > 0;
    577    }
    578
    579    /**
    580     * Gets whether the tokenizer currently returns empty tokens as null.
    581     * The default for this property is false.
    582     *
    583     * @return true if empty tokens are returned as null
    584     */
    585    public boolean isEmptyTokenAsNull() {
    586        return this.emptyAsNull;
    587    }
    588
    589    /**
    590     * Gets whether the tokenizer currently ignores empty tokens.
    591     * The default for this property is true.
    592     *
    593     * @return true if empty tokens are not returned
    594     */
    595    public boolean isIgnoreEmptyTokens() {
    596        return ignoreEmptyTokens;
    597    }
    598
    599    /**
    600     * Checks if the characters at the index specified match the quote
    601     * already matched in readNextToken().
    602     *
    603     * @param srcChars  the character array being tokenized
    604     * @param pos  the position to check for a quote
    605     * @param len  the length of the character array being tokenized
    606     * @param quoteStart  the start position of the matched quote, 0 if no quoting
    607     * @param quoteLen  the length of the matched quote, 0 if no quoting
    608     * @return true if a quote is matched
    609     */
    610    private boolean isQuote(final char[] srcChars, final int pos, final int len, final int quoteStart, final int quoteLen) {
    611        for (int i = 0; i < quoteLen; i++) {
    612            if (pos + i >= len || srcChars[pos + i] != srcChars[quoteStart + i]) {
    613                return false;
    614            }
    615        }
    616        return true;
    617    }
    618
    619    /**
    620     * Gets the next token.
    621     *
    622     * @return the next String token
    623     * @throws NoSuchElementException if there are no more elements
    624     */
    625    @Override
    626    public String next() {
    627        if (hasNext()) {
    628            return tokens[tokenPos++];
    629        }
    630        throw new NoSuchElementException();
    631    }
    632
    633    /**
    634     * Gets the index of the next token to return.
    635     *
    636     * @return the next token index
    637     */
    638    @Override
    639    public int nextIndex() {
    640        return tokenPos;
    641    }
    642
    643    /**
    644     * Gets the next token from the String.
    645     * Equivalent to {@link #next()} except it returns null rather than
    646     * throwing {@link NoSuchElementException} when no tokens remain.
    647     *
    648     * @return the next sequential token, or null when no more tokens are found
    649     */
    650    public String nextToken() {
    651        if (hasNext()) {
    652            return tokens[tokenPos++];
    653        }
    654        return null;
    655    }
    656
    657    /**
    658     * Gets the token previous to the last returned token.
    659     *
    660     * @return the previous token
    661     */
    662    @Override
    663    public String previous() {
    664        if (hasPrevious()) {
    665            return tokens[--tokenPos];
    666        }
    667        throw new NoSuchElementException();
    668    }
    669
    670    /**
    671     * Gets the index of the previous token.
    672     *
    673     * @return the previous token index
    674     */
    675    @Override
    676    public int previousIndex() {
    677        return tokenPos - 1;
    678    }
    679
    680    /**
    681     * Gets the previous token from the String.
    682     *
    683     * @return the previous sequential token, or null when no more tokens are found
    684     */
    685    public String previousToken() {
    686        if (hasPrevious()) {
    687            return tokens[--tokenPos];
    688        }
    689        return null;
    690    }
    691
    692    /**
    693     * Reads character by character through the String to get the next token.
    694     *
    695     * @param srcChars  the character array being tokenized
    696     * @param start  the first character of field
    697     * @param len  the length of the character array being tokenized
    698     * @param workArea  a temporary work area
    699     * @param tokenList  the list of parsed tokens
    700     * @return the starting position of the next field (the character
    701     *  immediately after the delimiter), or -1 if end of string found
    702     */
    703    private int readNextToken(final char[] srcChars, int start, final int len, final StrBuilder workArea, final List<String> tokenList) {
    704        // skip all leading whitespace, unless it is the
    705        // field delimiter or the quote character
    706        while (start < len) {
    707            final int removeLen = Math.max(
    708                    getIgnoredMatcher().isMatch(srcChars, start, start, len),
    709                    getTrimmerMatcher().isMatch(srcChars, start, start, len));
    710            if (removeLen == 0 ||
    711                getDelimiterMatcher().isMatch(srcChars, start, start, len) > 0 ||
    712                getQuoteMatcher().isMatch(srcChars, start, start, len) > 0) {
    713                break;
    714            }
    715            start += removeLen;
    716        }
    717
    718        // handle reaching end
    719        if (start >= len) {
    720            addToken(tokenList, StringUtils.EMPTY);
    721            return -1;
    722        }
    723
    724        // handle empty token
    725        final int delimLen = getDelimiterMatcher().isMatch(srcChars, start, start, len);
    726        if (delimLen > 0) {
    727            addToken(tokenList, StringUtils.EMPTY);
    728            return start + delimLen;
    729        }
    730
    731        // handle found token
    732        final int quoteLen = getQuoteMatcher().isMatch(srcChars, start, start, len);
    733        if (quoteLen > 0) {
    734            return readWithQuotes(srcChars, start + quoteLen, len, workArea, tokenList, start, quoteLen);
    735        }
    736        return readWithQuotes(srcChars, start, len, workArea, tokenList, 0, 0);
    737    }
    738
    739    /**
    740     * Reads a possibly quoted string token.
    741     *
    742     * @param srcChars  the character array being tokenized
    743     * @param start  the first character of field
    744     * @param len  the length of the character array being tokenized
    745     * @param workArea  a temporary work area
    746     * @param tokenList  the list of parsed tokens
    747     * @param quoteStart  the start position of the matched quote, 0 if no quoting
    748     * @param quoteLen  the length of the matched quote, 0 if no quoting
    749     * @return the starting position of the next field (the character
    750     *  immediately after the delimiter, or if end of string found,
    751     *  then the length of string
    752     */
    753    private int readWithQuotes(final char[] srcChars, final int start, final int len, final StrBuilder workArea,
    754                               final List<String> tokenList, final int quoteStart, final int quoteLen) {
    755        // Loop until we've found the end of the quoted
    756        // string or the end of the input
    757        workArea.clear();
    758        int pos = start;
    759        boolean quoting = quoteLen > 0;
    760        int trimStart = 0;
    761
    762        while (pos < len) {
    763            // quoting mode can occur several times throughout a string
    764            // we must switch between quoting and non-quoting until we
    765            // encounter a non-quoted delimiter, or end of string
    766            if (quoting) {
    767                // In quoting mode
    768
    769                // If we've found a quote character, see if it's
    770                // followed by a second quote.  If so, then we need
    771                // to actually put the quote character into the token
    772                // rather than end the token.
    773                if (isQuote(srcChars, pos, len, quoteStart, quoteLen)) {
    774                    if (isQuote(srcChars, pos + quoteLen, len, quoteStart, quoteLen)) {
    775                        // matched pair of quotes, thus an escaped quote
    776                        workArea.append(srcChars, pos, quoteLen);
    777                        pos += quoteLen * 2;
    778                        trimStart = workArea.size();
    779                        continue;
    780                    }
    781
    782                    // end of quoting
    783                    quoting = false;
    784                    pos += quoteLen;
    785                    continue;
    786                }
    787
    788            } else {
    789                // Not in quoting mode
    790
    791                // check for delimiter, and thus end of token
    792                final int delimLen = getDelimiterMatcher().isMatch(srcChars, pos, start, len);
    793                if (delimLen > 0) {
    794                    // return condition when end of token found
    795                    addToken(tokenList, workArea.substring(0, trimStart));
    796                    return pos + delimLen;
    797                }
    798
    799                // check for quote, and thus back into quoting mode
    800                if (quoteLen > 0 && isQuote(srcChars, pos, len, quoteStart, quoteLen)) {
    801                    quoting = true;
    802                    pos += quoteLen;
    803                    continue;
    804                }
    805
    806                // check for ignored (outside quotes), and ignore
    807                final int ignoredLen = getIgnoredMatcher().isMatch(srcChars, pos, start, len);
    808                if (ignoredLen > 0) {
    809                    pos += ignoredLen;
    810                    continue;
    811                }
    812
    813                // check for trimmed character
    814                // don't yet know if it's at the end, so copy to workArea
    815                // use trimStart to keep track of trim at the end
    816                final int trimmedLen = getTrimmerMatcher().isMatch(srcChars, pos, start, len);
    817                if (trimmedLen > 0) {
    818                    workArea.append(srcChars, pos, trimmedLen);
    819                    pos += trimmedLen;
    820                    continue;
    821                }
    822            }
    823            // copy regular character from inside quotes
    824            workArea.append(srcChars[pos++]);
    825            trimStart = workArea.size();
    826        }
    827
    828        // return condition when end of string found
    829        addToken(tokenList, workArea.substring(0, trimStart));
    830        return -1;
    831    }
    832
    833    /**
    834     * Unsupported ListIterator operation.
    835     *
    836     * @throws UnsupportedOperationException always
    837     */
    838    @Override
    839    public void remove() {
    840        throw new UnsupportedOperationException("remove() is unsupported");
    841    }
    842
    843    /**
    844     * Resets this tokenizer, forgetting all parsing and iteration already completed.
    845     * <p>
    846     * This method allows the same tokenizer to be reused for the same String.
    847     * </p>
    848     *
    849     * @return this, to enable chaining
    850     */
    851    public StrTokenizer reset() {
    852        tokenPos = 0;
    853        tokens = null;
    854        return this;
    855    }
    856
    857    /**
    858     * Reset this tokenizer, giving it a new input string to parse.
    859     * In this manner you can re-use a tokenizer with the same settings
    860     * on multiple input lines.
    861     *
    862     * @param input  the new character array to tokenize, not cloned, null sets no text to parse
    863     * @return this, to enable chaining
    864     */
    865    public StrTokenizer reset(final char[] input) {
    866        reset();
    867        this.chars = ArrayUtils.clone(input);
    868        return this;
    869    }
    870
    871    /**
    872     * Reset this tokenizer, giving it a new input string to parse.
    873     * In this manner you can re-use a tokenizer with the same settings
    874     * on multiple input lines.
    875     *
    876     * @param input  the new string to tokenize, null sets no text to parse
    877     * @return this, to enable chaining
    878     */
    879    public StrTokenizer reset(final String input) {
    880        reset();
    881        if (input != null) {
    882            this.chars = input.toCharArray();
    883        } else {
    884            this.chars = null;
    885        }
    886        return this;
    887    }
    888
    889    /**
    890     * Unsupported ListIterator operation.
    891     * @param obj this parameter ignored.
    892     * @throws UnsupportedOperationException always
    893     */
    894    @Override
    895    public void set(final String obj) {
    896        throw new UnsupportedOperationException("set() is unsupported");
    897    }
    898
    899    /**
    900     * Sets the field delimiter character.
    901     *
    902     * @param delim  the delimiter character to use
    903     * @return this, to enable chaining
    904     */
    905    public StrTokenizer setDelimiterChar(final char delim) {
    906        return setDelimiterMatcher(StrMatcher.charMatcher(delim));
    907    }
    908
    909    /**
    910     * Sets the field delimiter matcher.
    911     * <p>
    912     * The delimiter is used to separate one token from another.
    913     * </p>
    914     *
    915     * @param delim  the delimiter matcher to use
    916     * @return this, to enable chaining
    917     */
    918    public StrTokenizer setDelimiterMatcher(final StrMatcher delim) {
    919        if (delim == null) {
    920            this.delimMatcher = StrMatcher.noneMatcher();
    921        } else {
    922            this.delimMatcher = delim;
    923        }
    924        return this;
    925    }
    926
    927    /**
    928     * Sets the field delimiter string.
    929     *
    930     * @param delim  the delimiter string to use
    931     * @return this, to enable chaining
    932     */
    933    public StrTokenizer setDelimiterString(final String delim) {
    934        return setDelimiterMatcher(StrMatcher.stringMatcher(delim));
    935    }
    936
    937    /**
    938     * Sets whether the tokenizer should return empty tokens as null.
    939     * The default for this property is false.
    940     *
    941     * @param emptyAsNull  whether empty tokens are returned as null
    942     * @return this, to enable chaining
    943     */
    944    public StrTokenizer setEmptyTokenAsNull(final boolean emptyAsNull) {
    945        this.emptyAsNull = emptyAsNull;
    946        return this;
    947    }
    948
    949    /**
    950     * Sets the character to ignore.
    951     * <p>
    952     * This character is ignored when parsing the String, unless it is
    953     * within a quoted region.
    954     *
    955     * @param ignored  the ignored character to use
    956     * @return this, to enable chaining
    957     */
    958    public StrTokenizer setIgnoredChar(final char ignored) {
    959        return setIgnoredMatcher(StrMatcher.charMatcher(ignored));
    960    }
    961
    962    /**
    963     * Sets the matcher for characters to ignore.
    964     * <p>
    965     * These characters are ignored when parsing the String, unless they are
    966     * within a quoted region.
    967     * </p>
    968     *
    969     * @param ignored  the ignored matcher to use, null ignored
    970     * @return this, to enable chaining
    971     */
    972    public StrTokenizer setIgnoredMatcher(final StrMatcher ignored) {
    973        if (ignored != null) {
    974            this.ignoredMatcher = ignored;
    975        }
    976        return this;
    977    }
    978
    979    /**
    980     * Sets whether the tokenizer should ignore and not return empty tokens.
    981     * The default for this property is true.
    982     *
    983     * @param ignoreEmptyTokens  whether empty tokens are not returned
    984     * @return this, to enable chaining
    985     */
    986    public StrTokenizer setIgnoreEmptyTokens(final boolean ignoreEmptyTokens) {
    987        this.ignoreEmptyTokens = ignoreEmptyTokens;
    988        return this;
    989    }
    990
    991    /**
    992     * Sets the quote character to use.
    993     * <p>
    994     * The quote character is used to wrap data between the tokens.
    995     * This enables delimiters to be entered as data.
    996     * </p>
    997     *
    998     * @param quote  the quote character to use
    999     * @return this, to enable chaining
    1000     */
    1001    public StrTokenizer setQuoteChar(final char quote) {
    1002        return setQuoteMatcher(StrMatcher.charMatcher(quote));
    1003    }
    1004
    1005    /**
    1006     * Sets the quote matcher to use.
    1007     * <p>
    1008     * The quote character is used to wrap data between the tokens.
    1009     * This enables delimiters to be entered as data.
    1010     * </p>
    1011     *
    1012     * @param quote  the quote matcher to use, null ignored
    1013     * @return this, to enable chaining
    1014     */
    1015    public StrTokenizer setQuoteMatcher(final StrMatcher quote) {
    1016        if (quote != null) {
    1017            this.quoteMatcher = quote;
    1018        }
    1019        return this;
    1020    }
    1021
    1022    /**
    1023     * Sets the matcher for characters to trim.
    1024     * <p>
    1025     * These characters are trimmed off on each side of the delimiter
    1026     * until the token or quote is found.
    1027     * </p>
    1028     *
    1029     * @param trimmer  the trimmer matcher to use, null ignored
    1030     * @return this, to enable chaining
    1031     */
    1032    public StrTokenizer setTrimmerMatcher(final StrMatcher trimmer) {
    1033        if (trimmer != null) {
    1034            this.trimmerMatcher = trimmer;
    1035        }
    1036        return this;
    1037    }
    1038
    1039    // API
    1040    /**
    1041     * Gets the number of tokens found in the String.
    1042     *
    1043     * @return the number of matched tokens
    1044     */
    1045    public int size() {
    1046        checkTokenized();
    1047        return tokens.length;
    1048    }
    1049
    1050    /**
    1051     * Internal method to performs the tokenization.
    1052     * <p>
    1053     * Most users of this class do not need to call this method. This method
    1054     * will be called automatically by other (public) methods when required.
    1055     * </p>
    1056     * <p>
    1057     * This method exists to allow subclasses to add code before or after the
    1058     * tokenization. For example, a subclass could alter the character array,
    1059     * offset or count to be parsed, or call the tokenizer multiple times on
    1060     * multiple strings. It is also be possible to filter the results.
    1061     * </p>
    1062     * <p>
    1063     * {@link StrTokenizer} will always pass a zero offset and a count
    1064     * equal to the length of the array to this method, however a subclass
    1065     * may pass other values, or even an entirely different array.
    1066     * </p>
    1067     *
    1068     * @param srcChars  the character array being tokenized, may be null
    1069     * @param offset  the start position within the character array, must be valid
    1070     * @param count  the number of characters to tokenize, must be valid
    1071     * @return the modifiable list of String tokens, unmodifiable if null array or zero count
    1072     */
    1073    protected List<String> tokenize(final char[] srcChars, final int offset, final int count) {
    1074        if (ArrayUtils.isEmpty(srcChars)) {
    1075            return Collections.emptyList();
    1076        }
    1077        final StrBuilder buf = new StrBuilder();
    1078        final List<String> tokenList = new ArrayList<>();
    1079        int pos = offset;
    1080
    1081        // loop around the entire buffer
    1082        while (pos >= 0 && pos < count) {
    1083            // find next token
    1084            pos = readNextToken(srcChars, pos, count, buf, tokenList);
    1085
    1086            // handle case where end of string is a delimiter
    1087            if (pos >= count) {
    1088                addToken(tokenList, StringUtils.EMPTY);
    1089            }
    1090        }
    1091        return tokenList;
    1092    }
    1093
    1094    /**
    1095     * Gets the String content that the tokenizer is parsing.
    1096     *
    1097     * @return the string content being parsed
    1098     */
    1099    @Override
    1100    public String toString() {
    1101        if (tokens == null) {
    1102            return "StrTokenizer[not tokenized yet]";
    1103        }
    1104        return "StrTokenizer" + getTokenList();
    1105    }
    1106
    1107}