Package org.crsh.cli.impl.tokenizer

Examples of org.crsh.cli.impl.tokenizer.Token$Literal$Option


  public static void main(String[] args) throws Exception {
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
   
    Option inputOpt = DefaultOptionCreator.inputOption().create();
   
    Option outputOpt = DefaultOptionCreator.outputOption().create();
   
    Option labelOpt = obuilder.withLongName("label").withRequired(true).withArgument(
      abuilder.withName("label").withMinimum(1).withMaximum(1).create()).withDescription("The label of the file")
        .withShortName("l").create();
   
    Option analyzerOpt = obuilder.withLongName("analyzer").withArgument(
      abuilder.withName("analyzer").withMinimum(1).withMaximum(1).create()).withDescription(
      "The fully qualified class name of the analyzer to use. "
          + "Must have a no-arg constructor.  Default is the StandardAnalyzer").withShortName("a").create();
   
    Option charsetOpt = obuilder.withLongName("charset").withArgument(
      abuilder.withName("charset").withMinimum(1).withMaximum(1).create()).withDescription(
      "The character encoding of the input file").withShortName("c").create();
   
    Option collapseOpt = obuilder.withLongName("collapse").withRequired(true).withArgument(
      abuilder.withName("collapse").withMinimum(1).withMaximum(1).create()).withDescription(
      "Collapse a whole directory to a single file, one doc per line").withShortName("p").create();
   
    Option helpOpt = DefaultOptionCreator.helpOption();
    Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt).withOption(labelOpt)
        .withOption(analyzerOpt).withOption(charsetOpt).withOption(collapseOpt).withOption(helpOpt).create();
    try {
      Parser parser = new Parser();
      parser.setGroup(group);
View Full Code Here


  }

  public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    GroupBuilder gbuilder = new GroupBuilder();

    Option inputOpt = DefaultOptionCreator.inputOption().withRequired(false).create();
    Option outputOpt = DefaultOptionCreator.outputOption().withRequired(false).create();
    Option helpOpt = DefaultOptionCreator.helpOption();
    Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt).withOption(helpOpt).create();

    try {
      Parser parser = new Parser();
      parser.setGroup(group);
View Full Code Here

  }

  boolean parseArgs(String[] args) {
    DefaultOptionBuilder builder = new DefaultOptionBuilder();

    Option help = builder.withLongName("help").withDescription("print this list").create();

    ArgumentBuilder argumentBuilder = new ArgumentBuilder();
    Option inputFileOption = builder.withLongName("input")
            .withRequired(true)
            .withArgument(argumentBuilder.withName("input").withMaximum(1).create())
            .withDescription("where to get training data")
            .create();

    Option modelFileOption = builder.withLongName("model")
            .withRequired(true)
            .withArgument(argumentBuilder.withName("model").withMaximum(1).create())
            .withDescription("where to get a model")
            .create();
View Full Code Here

 
  public static void main(String[] args) throws Exception {
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
    Option helpOpt = DefaultOptionCreator.helpOption();
   
    Option parentOpt = obuilder.withLongName("parent").withRequired(true).withArgument(
      abuilder.withName("parent").withMinimum(1).withMaximum(1).create()).withDescription(
      "Parent dir containing the newsgroups").withShortName("p").create();
   
    Option outputDirOpt = obuilder.withLongName("outputDir").withRequired(true).withArgument(
      abuilder.withName("outputDir").withMinimum(1).withMaximum(1).create()).withDescription(
      "The output directory").withShortName("o").create();
   
    Option analyzerNameOpt = obuilder.withLongName("analyzerName").withRequired(true).withArgument(
      abuilder.withName("analyzerName").withMinimum(1).withMaximum(1).create()).withDescription(
      "The class name of the analyzer").withShortName("a").create();
   
    Option charsetOpt = obuilder.withLongName("charset").withRequired(true).withArgument(
      abuilder.withName("charset").withMinimum(1).withMaximum(1).create()).withDescription(
      "The name of the character encoding of the input files").withShortName("c").create();
   
    Group group = gbuilder.withName("Options").withOption(analyzerNameOpt).withOption(charsetOpt).withOption(
      outputDirOpt).withOption(parentOpt).withOption(helpOpt).create();
View Full Code Here

  public static void main(String[] args) throws IOException {
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
   
    Option dumpFileOpt = obuilder.withLongName("dumpFile").withRequired(true).withArgument(
      abuilder.withName("dumpFile").withMinimum(1).withMaximum(1).create()).withDescription(
      "The path to the wikipedia dump file (.bz2 or uncompressed)").withShortName("d").create();
   
    Option outputDirOpt = obuilder.withLongName("outputDir").withRequired(true).withArgument(
      abuilder.withName("outputDir").withMinimum(1).withMaximum(1).create()).withDescription(
      "The output directory to place the splits in:\n"
          + "local files:\n\t/var/data/wikipedia-xml-chunks or\n\tfile:///var/data/wikipedia-xml-chunks\n"
          + "Hadoop DFS:\n\thdfs://wikipedia-xml-chunks\n"
          + "AWS S3 (blocks):\n\ts3://bucket-name/wikipedia-xml-chunks\n"
          + "AWS S3 (native files):\n\ts3n://bucket-name/wikipedia-xml-chunks\n")

    .withShortName("o").create();
   
    Option s3IdOpt = obuilder.withLongName("s3ID").withRequired(false).withArgument(
      abuilder.withName("s3Id").withMinimum(1).withMaximum(1).create()).withDescription("Amazon S3 ID key")
        .withShortName("i").create();
    Option s3SecretOpt = obuilder.withLongName("s3Secret").withRequired(false).withArgument(
      abuilder.withName("s3Secret").withMinimum(1).withMaximum(1).create()).withDescription(
      "Amazon S3 secret key").withShortName("s").create();
   
    Option chunkSizeOpt = obuilder.withLongName("chunkSize").withRequired(true).withArgument(
      abuilder.withName("chunkSize").withMinimum(1).withMaximum(1).create()).withDescription(
      "The Size of the chunk, in megabytes").withShortName("c").create();
    Option numChunksOpt = obuilder
        .withLongName("numChunks")
        .withRequired(false)
        .withArgument(abuilder.withName("numChunks").withMinimum(1).withMaximum(1).create())
        .withDescription(
          "The maximum number of chunks to create.  If specified, program will only create a subset of the chunks")
View Full Code Here

  public static void main(String[] args) throws Exception {
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
   
    Option inputOpt = DefaultOptionCreator.inputOption().create();
   
    Option dictOpt = obuilder.withLongName("dict").withRequired(true).withArgument(
      abuilder.withName("dict").withMinimum(1).withMaximum(1).create()).withDescription(
      "Dictionary to read in, in the same format as one created by "
          + "org.apache.mahout.utils.vectors.lucene.Driver").withShortName("d").create();
   
    Option outOpt = DefaultOptionCreator.outputOption().create();
   
    Option wordOpt = obuilder.withLongName("words").withRequired(false).withArgument(
      abuilder.withName("words").withMinimum(0).withMaximum(1).withDefault("20").create()).withDescription(
      "Number of words to print").withShortName("w").create();
    Option dictTypeOpt = obuilder.withLongName("dictionaryType").withRequired(false).withArgument(
      abuilder.withName("dictionaryType").withMinimum(1).withMaximum(1).create()).withDescription(
      "The dictionary file type (text|sequencefile)").withShortName("dt").create();
    Option helpOpt = obuilder.withLongName("help").withDescription("Print out help").withShortName("h")
        .create();
   
    Group group = gbuilder.withName("Options").withOption(dictOpt).withOption(outOpt).withOption(wordOpt)
        .withOption(inputOpt).withOption(dictTypeOpt).create();
    try {
View Full Code Here

  static class ReadingOption extends Status {

    <T> Response<T> process(Request<T> req) {
      Response<T> response = new Response<T>();
      Token token = req.tokenizer.peek();
      if (token == null) {
        response.add(new Event.Stop.Done(req.tokenizer.getIndex()));
      } else if (token instanceof Token.Whitespace) {
        response.add(new Event.Separator((Token.Whitespace) token));
        req.tokenizer.next();
      } else {
        Token.Literal literal = (Token.Literal)token;
        if (literal instanceof Token.Literal.Option) {
          Token.Literal.Option optionToken = (Token.Literal.Option)literal;
          if (optionToken.getName().length() == 0 && optionToken instanceof Token.Literal.Option.Long) {
            req.tokenizer.next();
            if (req.tokenizer.hasNext()) {
              response.status = new Status.WantReadArg();
            } else {
              if (req.mode == Mode.INVOKE) {
                response.status = new Status.Done();
                response.add(new Event.Stop.Done(req.tokenizer.getIndex()));
              } else {
                response.add(new Event.Stop.Unresolved.NoSuchOption(optionToken));
              }
            }
          } else {
            OptionDescriptor desc = req.command.resolveOption(literal.getValue());
            if (desc != null) {
              req.tokenizer.next();
              int arity = desc.getArity();
              LinkedList<Token.Literal.Word> values = new LinkedList<Token.Literal.Word>();
              while (arity > 0) {
                if (req.tokenizer.hasNext()) {
                  Token a = req.tokenizer.peek();
                  if (a instanceof Token.Whitespace) {
                    req.tokenizer.next();
                    if (req.tokenizer.hasNext() && req.tokenizer.peek() instanceof Token.Literal.Word) {
                      // ok
                    } else {
View Full Code Here

  static class ComputeArg extends Status {

    @Override
    <T> Response<T> process(Request<T> req) {
      Token token = req.tokenizer.peek();
      Response<T> response = new Response<T>();
      if (token == null) {
        response.add(new Event.Stop.Done(req.tokenizer.getIndex()));
      } else if (token instanceof Token.Whitespace) {
        response.add(new Event.Separator((Token.Whitespace) token));
        req.tokenizer.next();
      } else {

        //
        List<? extends ArgumentDescriptor> arguments = req.command.getArguments();

        // Count the number ok remaining non whitespace;
        int tokenCount = 0;
        int wordCount = 0;
        do {
          Token t = req.tokenizer.next();
          if (t instanceof Token.Literal) {
            wordCount++;
          }
          tokenCount++;
        }
        while (req.tokenizer.hasNext());
        req.tokenizer.pushBack(tokenCount);

        //
        int oneCount = 0;
        int zeroOrOneCount = 0;
        int index = 0;
        for (ArgumentDescriptor argument : arguments) {
          Multiplicity multiplicity = argument.getMultiplicity();
          if (multiplicity == Multiplicity.SINGLE) {
            if (argument.isRequired()) {
              if (oneCount + 1 > wordCount) {
                break;
              }
              oneCount++;
            } else {
              zeroOrOneCount++;
            }
          }
          index++;
        }

        // This the number of arguments we can satisfy
        arguments = arguments.subList(0, index);

        // How many words we can consume for zeroOrOne and zeroOrMore
        int toConsume = wordCount - oneCount;

        // Correct the zeroOrOneCount and adjust toConsume
        zeroOrOneCount = Math.min(zeroOrOneCount, toConsume);
        toConsume -= zeroOrOneCount;

        // The remaining
        LinkedList<Event> events = new LinkedList<Event>();
        for (ArgumentDescriptor argument : arguments) {
          int size;
          switch (argument.getMultiplicity()) {
            case SINGLE:
              if (argument.isRequired()) {
                size = 1;
              } else {
                if (zeroOrOneCount > 0) {
                  zeroOrOneCount--;
                  size = 1;
                } else {
                  size = 0;
                }
              }
              break;
            case MULTI:
              // We consume the remaining
              size = toConsume;
              toConsume = 0;
              break;
            default:
              throw new AssertionError();
          }

          // Now take care of the argument
          if (size > 0) {
            List<Token.Literal> values = new ArrayList<Token.Literal>(size);
            while (size > 0) {
              Token t = req.tokenizer.next();
              if (t instanceof Token.Literal) {
                values.add(((Token.Literal)t));
                size--;
              }
            }
View Full Code Here

      return new ReadingArg(index + 1);
    }

    @Override
    <T> Response<T> process(Request<T> req) {
      Token token = req.tokenizer.peek();
      Response<T> response = new Response<T>();
      if (token == null) {
        response.add(new Event.Stop.Done(req.tokenizer.getIndex()));
      } else if (token instanceof Token.Whitespace) {
        response.add(new Event.Separator((Token.Whitespace) token));
        req.tokenizer.next();
      } else {
        final Token.Literal literal = (Token.Literal)token;
        List<? extends ArgumentDescriptor> arguments = req.command.getArguments();
        if (index < arguments.size()) {
          ArgumentDescriptor argument = arguments.get(index);
          switch (argument.getMultiplicity()) {
            case SINGLE:
              req.tokenizer.next();
              response.add(new Event.Argument(req.command, argument, Arrays.asList(literal)));
              response.status = next();
              break;
            case MULTI:
              req.tokenizer.next();
              List<Token.Literal> values = new ArrayList<Token.Literal>();
              values.add(literal);
              while (req.tokenizer.hasNext()) {
                Token capture = req.tokenizer.next();
                if (capture instanceof Token.Literal) {
                  values.add(((Token.Literal)capture));
                } else {
                  if (req.tokenizer.hasNext()) {
                    // Ok
View Full Code Here

    }

    //
    StringBuilder rest = new StringBuilder();
    while (tokenizer.hasNext()) {
      Token token = tokenizer.next();
      rest.append(token.getRaw());
    }
    current.setRest(rest.toString());

    //
    return current;
View Full Code Here

TOP

Related Classes of org.crsh.cli.impl.tokenizer.Token$Literal$Option

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.