Examples of org.apache.commons.lang.text.StrTokenizer

org.apache.commons.lang.text.StrTokenizer
Tokenizes a string based based on delimiters (separators) and supporting quoting and ignored character concepts.
This class can split a String into many smaller strings. It aims to do a similar job to {@link java.util.StringTokenizer StringTokenizer}, however it offers much more control and flexibility including implementing the ListIterator interface. By default, it is set up like StringTokenizer.
The input String is split into a number of tokens. Each token is separated from the next String by a delimiter. One or more delimiter characters must be specified.
Each token may be surrounded by quotes. The quote matcher specifies the quote character(s). A quote may be escaped within a quoted section by duplicating itself.
Between each token and the delimiter are potentially characters that need trimming. The trimmer matcher specifies these characters. One usage might be to trim whitespace characters.
At any point outside the quotes there might potentially be invalid characters. The ignored matcher specifies these characters to be removed. One usage might be to remove new line characters.
Empty tokens may be removed or returned as null.
```
 "a,b,c"         - Three tokens "a","b","c"   (comma delimiter) " a, b , c "    - Three tokens "a","b","c"   (default CSV processing trims whitespace) "a, ", b ,", c" - Three tokens "a, " , " b ", ", c" (quoted text untouched) 
```
This tokenizer has the following properties and options:

Property Type Default

delim CharSetMatcher { \t\n\r\f}

quote NoneMatcher {}

ignore NoneMatcher {}

emptyTokenAsNull boolean false

ignoreEmptyTokens boolean true

@author Apache Software Foundation @author Matthew Inger @author Gary D. Gregory @since 2.2 @version $Id: StrTokenizer.java 907631 2010-02-08 12:22:48Z sebb $

Property	Type	Default
delim	CharSetMatcher	{ \t\n\r\f}
quote	NoneMatcher	{}
ignore	NoneMatcher	{}
emptyTokenAsNull	boolean	false
ignoreEmptyTokens	boolean	true

      if (log.isLoggable(Level.FINE)) 
         log.fine("parseMapping: the string to be mapped is '" + mappingText + "'");
      
      this.mapping = new Hashtable();
      // StringTokenizer tokenizer = new StringTokenizer(mappingText, ",");
      StrTokenizer tokenizer = new StrTokenizer(mappingText, ',', '"');
      XmlBlasterException ex = null;
      while (tokenizer.hasNext()) {
         String singleMapping = tokenizer.nextToken();
         int pos = singleMapping.indexOf("=");
         if (pos < 0)
            ex = new XmlBlasterException(this.glob, ErrorCode.RESOURCE_CONFIGURATION_PLUGINFAILED, ME, "syntax in xmlBlaster.properties for " + singleMapping +
                " is wrong: no equality sign between key and value");
//            ex = new XmlBlasterException(ME, "syntax in xmlBlaster.properties for " + singleMapping +

View Full Code Here

        return builder.toString();
    }


    public static final String[] stringToArray(String string)
    {
        StrTokenizer tokenizer = new StrTokenizer(string, ',', '"');
        String[] array = new String[tokenizer.size()];


        for (int i = 0; i < array.length; i++)
        {
            array[i] = tokenizer.nextToken();
        }


        return array;
    }

View Full Code Here

   * @return JDBC connection parameters
   */
  protected static Properties propertiesFromString(String input) {
    if (input != null && !input.isEmpty()) {
      Properties result = new Properties();
      StrTokenizer propertyTokenizer = StrTokenizer.getCSVInstance(input);
      StrTokenizer valueTokenizer = StrTokenizer.getCSVInstance();
      valueTokenizer.setDelimiterChar('=');
      while (propertyTokenizer.hasNext()){
        valueTokenizer.reset(propertyTokenizer.nextToken());
        String[] values = valueTokenizer.getTokenArray();
        if (values.length==2) {
          result.put(values[0], values[1]);
        }
      }
      return result;

View Full Code Here


    // to collect and return feedback about erroneous use of filter
    // expressions
    String message = new String("");


    StrTokenizer tokenizer = new StrTokenizer(inFilter, ' ', '\"');


    // conjunctions collecting conditions
    Conjunction conjWorkPiece = null;
    Conjunction conjProjects = null;
    Conjunction conjSteps = null;
    Conjunction conjProcesses = null;
    Conjunction conjTemplates = null;
    Conjunction conjUsers = null;
    Conjunction conjStepProperties = null;
    Conjunction conjProcessProperties = null;


    // this is needed if we filter processes
    if (flagProcesses) {
      conjProjects = Restrictions.conjunction();
      limitToUserAccessRights(conjProjects);
      // in case nothing is set here it needs to be removed again
      // happens if user has admin rights
      if (conjProjects.toString().equals("()")) {
        conjProjects = null;
        flagSetCritProjects = true;
      }
    }


    // this is needed if we filter steps
    if (flagSteps) {
      conjSteps = Restrictions.conjunction();
      limitToUserAssignedSteps(conjSteps, stepOpenOnly, userAssignedStepsOnly);
      // in case nothing is set here conjunction needs to be set to null
      // again
      if (conjSteps.toString().equals("()")) {
        conjSteps = null;
      }
    }


    // this is needed for the template filter (true) and the undefined
    // processes filter (false) in any other case it needs to be null
    if (isTemplate != null) {
      conjProcesses = Restrictions.conjunction();
      if (!isTemplate) {
        conjProcesses.add(Restrictions.eq("istTemplate", Boolean.valueOf(false)));
      } else {
        conjProcesses.add(Restrictions.eq("istTemplate", Boolean.valueOf(true)));
      }
    }
    
    // this is needed for evaluating a filter string
    while (tokenizer.hasNext()) {
      String tok = tokenizer.nextToken().trim();


      if (tok.toLowerCase().startsWith(FilterString.PROCESSPROPERTY) || tok.toLowerCase().startsWith(FilterString.PROZESSEIGENSCHAFT)) {
        if (conjProcessProperties == null) {
          conjProcessProperties = Restrictions.conjunction();
        }

View Full Code Here

    /*
     * --------------------- zusätzliche Parameter neben dem Modulnamen -------------------
     */
    HashMap<String, Object> typeParameters = new HashMap<String, Object>();
    String schrittModuleName = inSchritt.getTypModulName();
    StrTokenizer tokenizer = new StrTokenizer(inSchritt.getTypModulName());
    int counter = 0;
    while (tokenizer.hasNext()) {
      String tok = (String) tokenizer.next();
      if (counter == 0)
        schrittModuleName = tok;
      else {
        if (tok.contains(":")) {
          String key = tok.split(":")[0];

View Full Code Here

    public void execute(List<Prozess> inProzesse, String inScript) {
        this.myParameters = new HashMap<String, String>();
        /*
         * -------------------------------- alle Suchparameter zerlegen und erfassen --------------------------------
         */
        StrTokenizer tokenizer = new StrTokenizer(inScript, ' ', '\"');
        while (tokenizer.hasNext()) {
            String tok = tokenizer.nextToken();
            if (tok.indexOf(":") == -1) {
                Helper.setFehlerMeldung("goobiScriptfield", "missing delimiter / unknown parameter: ", tok);
            } else {
                String myKey = tok.substring(0, tok.indexOf(":"));
                String myValue = tok.substring(tok.indexOf(":") + 1);

View Full Code Here

  /**
   * Parses a CharSequence into a list of values, all of some other type.
   */
  public static List<Object> parseList(CharBuffer chars, Type listItemType,
      String nullStr, String listDelim) throws ColumnParseException {
    StrTokenizer tokenizer = new StrTokenizer(chars.toString(), listDelim.charAt(0));
    List<Object> out = new ArrayList<Object>();


    while (tokenizer.hasNext()) {
      String part = (String) tokenizer.next();
      out.add(parseType(CharBuffer.wrap(part), listItemType, nullStr, listDelim));
    }


    return Collections.unmodifiableList(out);
  }

View Full Code Here

    protected String getDistLabel(String requestURI) {
        return getNthPiece(4, requestURI);
    }


    protected String getLastPiece(String requestURI) {
        StrTokenizer st = new StrTokenizer(decodeURL(requestURI), "/");
        List<String> tokens = st.getTokenList();
        if (tokens.size() < 1) {
            return "";
        }
        return tokens.get(tokens.size() - 1);
    }

View Full Code Here

    protected String getDistFilePath(String requestURI) {
        // Goal is we find where the distribution file path starts
        // then we return the entire path, it may include an unknown
        // level of sub-directories.


        StrTokenizer st = new StrTokenizer(decodeURL(requestURI), "/");
        List<String> tokens = st.getTokenList();
        if (tokens.isEmpty()) {
            return "";
        }
        int startIndex = 4;
        String distFilePath = "";

View Full Code Here

     * @param n nth element to return from requestURI, (first element corresponds to 1, not 0)
     * @param requestURI
     * 
     */
    protected String getNthPiece(int n, String requestURI) {
        StrTokenizer st = new StrTokenizer(decodeURL(requestURI), "/");
        List<String> tokens = st.getTokenList();
        if (tokens.size() < n) {
            return "";
        }
        return tokens.get(n - 1); // caller is starting at 1 not 0
    }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.commons.lang.text.StrTokenizer

ch.iterate.openstack.swift.handler.ContainerResponseHandler

com.dbdeploy.database.QueryStatementSplitter

com.odiago.flumebase.io.CharBufferUtils

com.rackspacecloud.client.cloudfiles.FilesClient

de.sub.goobi.forms.ModuleServerForm

de.sub.goobi.helper.GoobiScript

org.apache.felix.sigil.eclipse.preferences.PrefsUtils

org.apache.sqoop.mapreduce.db.DBConfiguration

org.criticalfailure.torchlight.core.calculators.object.BaseObjectPropertyCalculator

org.criticalfailure.torchlight.core.calculators.object.MapObjectPropertyCalculator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.