Examples of org.apache.commons.lang.text.StrTokenizer

org.apache.commons.lang.text.StrTokenizer
Tokenizes a string based based on delimiters (separators) and supporting quoting and ignored character concepts.
This class can split a String into many smaller strings. It aims to do a similar job to {@link java.util.StringTokenizer StringTokenizer}, however it offers much more control and flexibility including implementing the ListIterator interface. By default, it is set up like StringTokenizer.
The input String is split into a number of tokens. Each token is separated from the next String by a delimiter. One or more delimiter characters must be specified.
Each token may be surrounded by quotes. The quote matcher specifies the quote character(s). A quote may be escaped within a quoted section by duplicating itself.
Between each token and the delimiter are potentially characters that need trimming. The trimmer matcher specifies these characters. One usage might be to trim whitespace characters.
At any point outside the quotes there might potentially be invalid characters. The ignored matcher specifies these characters to be removed. One usage might be to remove new line characters.
Empty tokens may be removed or returned as null.
```
 "a,b,c"         - Three tokens "a","b","c"   (comma delimiter) " a, b , c "    - Three tokens "a","b","c"   (default CSV processing trims whitespace) "a, ", b ,", c" - Three tokens "a, " , " b ", ", c" (quoted text untouched) 
```
This tokenizer has the following properties and options:

Property Type Default

delim CharSetMatcher { \t\n\r\f}

quote NoneMatcher {}

ignore NoneMatcher {}

emptyTokenAsNull boolean false

ignoreEmptyTokens boolean true

@author Apache Software Foundation @author Matthew Inger @author Gary D. Gregory @since 2.2 @version $Id: StrTokenizer.java 907631 2010-02-08 12:22:48Z sebb $

Property	Type	Default
delim	CharSetMatcher	{ \t\n\r\f}
quote	NoneMatcher	{}
ignore	NoneMatcher	{}
emptyTokenAsNull	boolean	false
ignoreEmptyTokens	boolean	true

    }


    protected List<String> gatherInputValues(ObjectProperty op, ObjectData od) {


        // get the input names
        StrTokenizer st = new StrTokenizer(op.getCalculatorInputs(), ",");
        st.setTrimmerMatcher(StrMatcher.trimMatcher());
        String[] calculatorInputs = st.getTokenArray();
        logger.trace("calculatorInputs: " + calculatorInputs);


        // gather values
        List<String> inputValues = new Vector<String>();
        for(String input : calculatorInputs) {

View Full Code Here

        Map<String, String> valueMap = new HashMap<String, String>();
        List<String> inputValues = gatherInputValues(op, od);
        logger.trace("inputValues: " + inputValues);


        // get list of mapped values
        StrTokenizer st = new StrTokenizer(op.getCalculatorBody(), ",");
        st.setTrimmerMatcher(StrMatcher.trimMatcher());
        String[] mapValues = st.getTokenArray();
        logger.trace("mapValues: " + mapValues);


        for(String mapItem : mapValues) {
            logger.trace("mapItem: " + mapItem);

View Full Code Here

            // get calculator inputs
            String inputs = StringUtils.stripToNull(op.getCalculatorInputs());
            logger.trace("inputs: " + inputs);


            if(inputs != null && inputs.length() > 0) {
                StrTokenizer st = new StrTokenizer(inputs, ",");
                st.setTrimmerMatcher(StrMatcher.trimMatcher());
                logger.trace("st: " + st);


                String[] calculatorInputs = st.getTokenArray();
                logger.trace("calculatorInputs: " + calculatorInputs);


                // check this property's inputs against the provided name
                for(String input : calculatorInputs) {
                    logger.trace("input: " + input);

View Full Code Here

        logger.trace("strippedString: " + strippedString);
        // make sure the string contains the start and end braces
        if(strippedString.startsWith("{") && strippedString.endsWith("}")) {
            logger.debug("parsing string into array");


            StrTokenizer tokenizer = new StrTokenizer(strippedString.substring(1, strippedString.length() - 1),
                    StrMatcher.commaMatcher(), StrMatcher.quoteMatcher());
            tokenizer.setTrimmerMatcher(StrMatcher.trimMatcher());
            setValues(tokenizer.getTokenList());
            return;
        }


        logger.debug("setting value list as a single value");
        List<String> values = new Vector<String>();

View Full Code Here

            throws ObjectPropertyValidationException {


        logger.debug("validating: " + value + "; property: " + op);


        // get list of valid values
        StrTokenizer st = new StrTokenizer(op.getValidatorBody(), ",");
        st.setTrimmerMatcher(StrMatcher.trimMatcher());
        String[] validValues = st.getTokenArray();
        logger.trace("validValues: " + validValues);


        for(String vv : validValues) {
            logger.trace("vv: " + vv);

View Full Code Here

   * @return JDBC connection parameters
   */
  protected static Properties propertiesFromString(String input) {
    if (input != null && !input.isEmpty()) {
      Properties result = new Properties();
      StrTokenizer propertyTokenizer = StrTokenizer.getCSVInstance(input);
      StrTokenizer valueTokenizer = StrTokenizer.getCSVInstance();
      valueTokenizer.setDelimiterChar('=');
      while (propertyTokenizer.hasNext()){
        valueTokenizer.reset(propertyTokenizer.nextToken());
        String[] values = valueTokenizer.getTokenArray();
        if (values.length==2) {
          result.put(values[0], values[1]);
        }
      }
      return result;

View Full Code Here

   * @return JDBC connection parameters
   */
  protected static Properties propertiesFromString(String input) {
    if (input != null && !input.isEmpty()) {
      Properties result = new Properties();
      StrTokenizer propertyTokenizer = StrTokenizer.getCSVInstance(input);
      StrTokenizer valueTokenizer = StrTokenizer.getCSVInstance();
      valueTokenizer.setDelimiterChar('=');
      while (propertyTokenizer.hasNext()){
        valueTokenizer.reset(propertyTokenizer.nextToken());
        String[] values = valueTokenizer.getTokenArray();
        if (values.length==2) {
          result.put(values[0], values[1]);
        }
      }
      return result;

View Full Code Here

        }
      
        if (response.getStatusCode() == HttpStatus.SC_OK)
        {
          // logger.warn(method.getResponseCharSet());
          StrTokenizer tokenize = new StrTokenizer(response.getResponseBodyAsString());
          tokenize.setDelimiterString("\n");
          String [] containers = tokenize.getTokenArray();          
          ArrayList <FilesContainer> containerList = new ArrayList<FilesContainer>();
          for(String container : containers) { 
            containerList.add(new FilesContainer(container, this));
          }
          return containerList;

View Full Code Here

            }
          }


          if (response.getStatusCode() == HttpStatus.SC_OK)
          {
            StrTokenizer tokenize = new StrTokenizer(response.getResponseBodyAsString());
            tokenize.setDelimiterString("\n");
            String [] containers = tokenize.getTokenArray();
            List<String> returnValue = new ArrayList<String>();
            for (String containerName: containers)
            {
              returnValue.add(containerName);
            }

View Full Code Here

    // we get one of these each time we encounter a w:dataBinding
    // element in a content control; pity it is not done just
    // once!
    
    // first tokenise on space
    StrTokenizer tokens = new StrTokenizer(prefixMappings);
    while (tokens.hasNext() ) {
      String token = tokens.nextToken();
      //log.debug("Got: " + token);
      int pos = token.indexOf("=");
      String prefix = token.substring(6, pos); // drop xmlns:
      //log.debug("Got: " + prefix);
      String uri = token.substring(pos+2, token.lastIndexOf("'"));

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.commons.lang.text.StrTokenizer

ch.iterate.openstack.swift.handler.ContainerResponseHandler

com.dbdeploy.database.QueryStatementSplitter

com.odiago.flumebase.io.CharBufferUtils

com.rackspacecloud.client.cloudfiles.FilesClient

de.sub.goobi.forms.ModuleServerForm

de.sub.goobi.helper.GoobiScript

org.apache.felix.sigil.eclipse.preferences.PrefsUtils

org.apache.sqoop.mapreduce.db.DBConfiguration

org.criticalfailure.torchlight.core.calculators.object.BaseObjectPropertyCalculator

org.criticalfailure.torchlight.core.calculators.object.MapObjectPropertyCalculator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.