Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.MapFile$Reader$Option

35
36
37
38
39
40
41
42
43
44
45
46
47
48
    ToolRunner.run(new TransposeJob(), args);
  }

  @Override
  public int run(String[] strings) throws Exception {
    Option numRowsOpt = buildOption("numRows",
                                    "nr",
                                    "Number of rows of the input matrix");
    Option numColsOpt = buildOption("numCols",
                                    "nc",
                                    "Number of columns of the input matrix");
    Map<String,String> parsedArgs = parseArguments(strings, numRowsOpt, numColsOpt);

    String inputPathString = parsedArgs.get("--input");
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    ToolRunner.run(new MatrixMultiplicationJob(), args);
  }

  @Override
  public int run(String[] strings) throws Exception {
    Option numRowsAOpt = buildOption("numRowsA",
                                     "nra",
                                     "Number of rows of the first input matrix");
    Option numColsAOpt = buildOption("numColsA",
                                     "nca",
                                     "Number of columns of the first input matrix");
    Option numRowsBOpt = buildOption("numRowsB",
                                     "nrb",
                                     "Number of rows of the second input matrix");

    Option numColsBOpt = buildOption("numColsB",
                                     "ncb",
                                     "Number of columns of the second input matrix");
    Option inputPathA = buildOption("inputPathA",
                                    "ia",
                                    "Path to the first input matrix");
    Option inputPathB = buildOption("inputPathB",
                                    "ib",
                                    "Path to the second input matrix");

    Map<String, String> argMap = parseArguments(strings,
                                                numRowsAOpt,
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
   
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
   
    Option pathOpt = obuilder.withLongName("path").withRequired(true).withArgument(
      abuilder.withName("path").withMinimum(1).withMaximum(1).create()).withDescription(
      "The local file system path").withShortName("m").create();
   
    Option classifyOpt = obuilder.withLongName("classify").withRequired(true).withArgument(
      abuilder.withName("classify").withMinimum(1).withMaximum(1).create()).withDescription(
      "The doc to classify").withShortName("").create();
   
    Option encodingOpt = obuilder.withLongName("encoding").withRequired(true).withArgument(
      abuilder.withName("encoding").withMinimum(1).withMaximum(1).create()).withDescription(
      "The file encoding.  Default: UTF-8").withShortName("e").create();
   
    Option analyzerOpt = obuilder.withLongName("analyzer").withRequired(true).withArgument(
      abuilder.withName("analyzer").withMinimum(1).withMaximum(1).create()).withDescription(
      "The Analyzer to use").withShortName("a").create();
   
    Option defaultCatOpt = obuilder.withLongName("defaultCat").withRequired(true).withArgument(
      abuilder.withName("defaultCat").withMinimum(1).withMaximum(1).create()).withDescription(
      "The default category").withShortName("d").create();
   
    Option gramSizeOpt = obuilder.withLongName("gramSize").withRequired(true).withArgument(
      abuilder.withName("gramSize").withMinimum(1).withMaximum(1).create()).withDescription(
      "Size of the n-gram").withShortName("ng").create();
   
    Option typeOpt = obuilder.withLongName("classifierType").withRequired(true).withArgument(
      abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).withDescription(
      "Type of classifier").withShortName("type").create();
   
    Option dataSourceOpt = obuilder.withLongName("dataSource").withRequired(true).withArgument(
      abuilder.withName("dataSource").withMinimum(1).withMaximum(1).create()).withDescription(
      "Location of model: hdfs|hbase").withShortName("source").create();
   
    Group options = gbuilder.withName("Options").withOption(pathOpt).withOption(classifyOpt).withOption(
      encodingOpt).withOption(analyzerOpt).withOption(defaultCatOpt).withOption(gramSizeOpt).withOption(
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
    }

    @Override
    public int run(String[] args) throws Exception {

      Option numRowsOpt = buildOption("numRows",
                                      "nr",
                                      "Number of rows of the input matrix");
      Option numColsOpt = buildOption("numCols",
                                      "nc",
                                      "Number of columns of the input matrix");
      Option desiredRankOpt = buildOption("rank",
                                          "r",
                                          "Desired decomposition rank (note: only roughly 1/4 to 1/3 "
                                        + "of these will have the top portion of the spectrum)");
      Option isSymmetricOpt = buildOption("symmetric",
                                          "sym",
                                          "Is the input matrix square and symmetric?");

      DistributedLanczosSolver.this.parsedArgs = parseArguments(args,
                                                                numRowsOpt,
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    return 0;
  }

  public Map<String,String> handleArgs(String[] args) {
    Option eigenInputOpt = buildOption("eigenInput", "ei",
        "The Path for purported eigenVector input files (SequenceFile<WritableComparable,VectorWritable>.", null);
    Option corpusInputOpt = buildOption("corpusInput", "ci",
        "The Path for corpus input files (SequenceFile<WritableComparable,VectorWritable>.");
    Option outOpt = DefaultOptionCreator.outputOption().create();
    Option helpOpt = DefaultOptionCreator.helpOption();
    Option inMemOpt = buildOption("inMemory", "mem", "Buffer eigen matrix into memory (if you have enough!)", "false");
    Option errorOpt = buildOption("maxError", "err", "Maximum acceptable error", "0.05");
    Option minEigenValOpt = buildOption("minEigenvalue", "mev", "Minimum eigenvalue to keep the vector for", "0.0");

    GroupBuilder gBuilder = new GroupBuilder().withName("Options")
                                              .withOption(eigenInputOpt)
                                              .withOption(corpusInputOpt)
                                              .withOption(helpOpt)
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
        .withArgument(arg).withDescription(description).create();
  }
 
  protected static Map<String,String> parseArguments(String[] args, Option... extraOpts) {
   
    Option inputOpt = DefaultOptionCreator.inputOption().create();
    Option tempDirOpt = buildOption("tempDir", "t", "Intermediate output directory", "temp");
    Option outputOpt = DefaultOptionCreator.outputOption().create();
    Option helpOpt = DefaultOptionCreator.helpOption();
    Option jarFileOpt = buildOption("jarFile", "m", "Implementation jar", false, null);
   
    GroupBuilder gBuilder = new GroupBuilder().withName("Options").withOption(inputOpt)
        .withOption(tempDirOpt).withOption(outputOpt).withOption(helpOpt).withOption(jarFileOpt);
   
    for (Option opt : extraOpts) {
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
  public static void main(String[] args) throws Exception {
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
   
    Option inputDirOpt = obuilder.withLongName("input").withRequired(true).withArgument(
      abuilder.withName("input").withMinimum(1).withMaximum(1).create()).withDescription(
      "input dir containing the documents in sequence file format").withShortName("i").create();
   
    Option outputDirOpt = obuilder.withLongName("output").withRequired(true).withArgument(
      abuilder.withName("output").withMinimum(1).withMaximum(1).create()).withDescription(
      "The output directory").withShortName("o").create();
    Option minSupportOpt = obuilder.withLongName("minSupport").withArgument(
      abuilder.withName("minSupport").withMinimum(1).withMaximum(1).create()).withDescription(
      "(Optional) Minimum Support. Default Value: 2").withShortName("s").create();
   
    Option analyzerNameOpt = obuilder.withLongName("analyzerName").withArgument(
      abuilder.withName("analyzerName").withMinimum(1).withMaximum(1).create()).withDescription(
      "The class name of the analyzer").withShortName("a").create();
   
    Option chunkSizeOpt = obuilder.withLongName("chunkSize").withArgument(
      abuilder.withName("chunkSize").withMinimum(1).withMaximum(1).create()).withDescription(
      "The chunkSize in MegaBytes. 100-10000 MB").withShortName("chunk").create();
   
    Option weightOpt = obuilder.withLongName("weight").withRequired(false).withArgument(
      abuilder.withName("weight").withMinimum(1).withMaximum(1).create()).withDescription(
      "The kind of weight to use. Currently TF or TFIDF").withShortName("wt").create();
   
    Option minDFOpt = obuilder.withLongName("minDF").withRequired(false).withArgument(
      abuilder.withName("minDF").withMinimum(1).withMaximum(1).create()).withDescription(
      "The minimum document frequency.  Default is 1").withShortName("md").create();
   
    Option maxDFPercentOpt = obuilder.withLongName("maxDFPercent").withRequired(false).withArgument(
      abuilder.withName("maxDFPercent").withMinimum(1).withMaximum(1).create()).withDescription(
      "The max percentage of docs for the DF.  Can be used to remove really high frequency terms."
          + " Expressed as an integer between 0 and 100. Default is 99.").withShortName("x").create();
   
    Option minLLROpt = obuilder.withLongName("minLLR").withRequired(false).withArgument(
      abuilder.withName("minLLR").withMinimum(1).withMaximum(1).create()).withDescription(
      "(Optional)The minimum Log Likelihood Ratio(Float)  Default is " + LLRReducer.DEFAULT_MIN_LLR)
        .withShortName("ml").create();
   
    Option numReduceTasksOpt = obuilder.withLongName("numReducers").withArgument(
      abuilder.withName("numReducers").withMinimum(1).withMaximum(1).create()).withDescription(
      "(Optional) Number of reduce tasks. Default Value: 1").withShortName("nr").create();
   
    Option powerOpt = obuilder.withLongName("norm").withRequired(false).withArgument(
      abuilder.withName("norm").withMinimum(1).withMaximum(1).create()).withDescription(
      "The norm to use, expressed as either a float or \"INF\" if you want to use the Infinite norm.  "
          + "Must be greater or equal to 0.  The default is not to normalize").withShortName("n").create();
    Option maxNGramSizeOpt = obuilder.withLongName("maxNGramSize").withRequired(false).withArgument(
      abuilder.withName("ngramSize").withMinimum(1).withMaximum(1).create())
        .withDescription(
          "(Optional) The maximum size of ngrams to create"
              + " (2 = bigrams, 3 = trigrams, etc) Default Value:2").withShortName("ng").create();
    Option sequentialAccessVectorOpt = obuilder.withLongName("sequentialAccessVector").withRequired(false)
        .withDescription(
          "(Optional) Whether output vectors should be SequentialAccessVectors If set true else false")
        .withShortName("seq").create();
   
    Option overwriteOutput = obuilder.withLongName("overwrite").withRequired(false).withDescription(
      "If set, overwrite the output directory").withShortName("w").create();
    Option helpOpt = obuilder.withLongName("help").withDescription("Print out help").withShortName("h")
        .create();
   
    Group group = gbuilder.withName("Options").withOption(minSupportOpt).withOption(analyzerNameOpt)
        .withOption(chunkSizeOpt).withOption(outputDirOpt).withOption(inputDirOpt).withOption(minDFOpt)
        .withOption(maxDFPercentOpt).withOption(weightOpt).withOption(powerOpt).withOption(minLLROpt)
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
  public static void main(String[] args) throws Exception {
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
   
    Option helpOpt = DefaultOptionCreator.helpOption();
   
    Option inputDirOpt = obuilder.withLongName("input").withRequired(true).withArgument(
      abuilder.withName("input").withMinimum(1).withMaximum(1).create()).withDescription(
      "The Directory on HDFS containing the collapsed, properly formatted files").withShortName("i").create();
   
    Option outputOpt = obuilder.withLongName("output").withRequired(true).withArgument(
      abuilder.withName("output").withMinimum(1).withMaximum(1).create()).withDescription(
      "The location of the modelon the HDFS").withShortName("o").create();
   
    Option gramSizeOpt = obuilder.withLongName("gramSize").withRequired(true).withArgument(
      abuilder.withName("gramSize").withMinimum(1).withMaximum(1).create()).withDescription(
      "Size of the n-gram. Default Value: 1 ").withShortName("ng").create();
   
    Option alphaOpt = obuilder.withLongName("alpha").withRequired(false).withArgument(
      abuilder.withName("a").withMinimum(1).withMaximum(1).create()).withDescription(
      "Smoothing parameter Default Value: 1.0").withShortName("a").create();
   
    Option typeOpt = obuilder.withLongName("classifierType").withRequired(true).withArgument(
      abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).withDescription(
      "Type of classifier: bayes|cbayes. Default: bayes").withShortName("type").create();
    Option dataSourceOpt = obuilder.withLongName("dataSource").withRequired(true).withArgument(
      abuilder.withName("dataSource").withMinimum(1).withMaximum(1).create()).withDescription(
      "Location of model: hdfs|hbase. Default Value: hdfs").withShortName("source").create();
   
    Group group = gbuilder.withName("Options").withOption(gramSizeOpt).withOption(helpOpt).withOption(
      inputDirOpt).withOption(outputOpt).withOption(typeOpt).withOption(dataSourceOpt).withOption(alphaOpt)
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
  public static void main(String[] args) throws IOException, InvalidDatastoreException {
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
   
    Option pathOpt = obuilder.withLongName("model").withRequired(true).withArgument(
      abuilder.withName("model").withMinimum(1).withMaximum(1).create()).withDescription(
      "The path on HDFS / Name of Hbase Table as defined by the -source parameter").withShortName("m")
        .create();
   
    Option dirOpt = obuilder.withLongName("testDir").withRequired(true).withArgument(
      abuilder.withName("testDir").withMinimum(1).withMaximum(1).create()).withDescription(
      "The directory where test documents resides in").withShortName("d").create();
   
    Option helpOpt = DefaultOptionCreator.helpOption();
   
    Option encodingOpt = obuilder.withLongName("encoding").withArgument(
      abuilder.withName("encoding").withMinimum(1).withMaximum(1).create()).withDescription(
      "The file encoding.  Defaults to UTF-8").withShortName("e").create();
   
    Option defaultCatOpt = obuilder.withLongName("defaultCat").withArgument(
      abuilder.withName("defaultCat").withMinimum(1).withMaximum(1).create()).withDescription(
      "The default category Default Value: unknown").withShortName("default").create();
   
    Option gramSizeOpt = obuilder.withLongName("gramSize").withRequired(true).withArgument(
      abuilder.withName("gramSize").withMinimum(1).withMaximum(1).create()).withDescription(
      "Size of the n-gram. Default Value: 1").withShortName("ng").create();
   
    Option alphaOpt = obuilder.withLongName("alpha").withRequired(false).withArgument(
      abuilder.withName("a").withMinimum(1).withMaximum(1).create()).withDescription(
      "Smoothing parameter Default Value: 1.0").withShortName("a").create();
   
    Option verboseOutputOpt = obuilder.withLongName("verbose").withRequired(false).withDescription(
      "Output which values were correctly and incorrectly classified").withShortName("v").create();
   
    Option typeOpt = obuilder.withLongName("classifierType").withRequired(true).withArgument(
      abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).withDescription(
      "Type of classifier: bayes|cbayes. Default Value: bayes").withShortName("type").create();
   
    Option dataSourceOpt = obuilder.withLongName("dataSource").withRequired(true).withArgument(
      abuilder.withName("dataSource").withMinimum(1).withMaximum(1).create()).withDescription(
      "Location of model: hdfs|hbase Default Value: hdfs").withShortName("source").create();
   
    Option methodOpt = obuilder.withLongName("method").withRequired(false).withArgument(
      abuilder.withName("method").withMinimum(1).withMaximum(1).create()).withDescription(
      "Method of Classification: sequential|mapreduce. Default Value: sequential").withShortName("method")
        .create();
   
    Group group = gbuilder.withName("Options").withOption(defaultCatOpt).withOption(dirOpt).withOption(
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
  public static void main(String[] args) throws Exception {
    DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
    ArgumentBuilder abuilder = new ArgumentBuilder();
    GroupBuilder gbuilder = new GroupBuilder();
   
    Option inputOpt = DefaultOptionCreator.inputOption().create();
    Option outputOpt = DefaultOptionCreator.outputOption().create();
    Option maxIterOpt = DefaultOptionCreator.maxIterOption().create();
    Option topicsOpt = DefaultOptionCreator.kOption().create();
    Option helpOpt = DefaultOptionCreator.helpOption();
   
    Option mOpt = obuilder.withLongName("alpha").withRequired(true).withShortName("m").withArgument(
      abuilder.withName("alpha").withMinimum(1).withMaximum(1).create()).withDescription(
      "The alpha0 value for the DirichletDistribution.").create();
   
    Option modelOpt = obuilder.withLongName("modelClass").withRequired(true).withShortName("d").withArgument(
      abuilder.withName("modelClass").withMinimum(1).withMaximum(1).create()).withDescription(
      "The ModelDistribution class name. "
          + "Defaults to org.apache.mahout.clustering.dirichlet.models.NormalModelDistribution").create();
   
    Option prototypeOpt = obuilder.withLongName("modelPrototypeClass").withRequired(false).withShortName("p")
        .withArgument(abuilder.withName("prototypeClass").withMinimum(1).withMaximum(1).create())
        .withDescription(
          "The ModelDistribution prototype Vector class name. "
              + "Defaults to org.apache.mahout.math.RandomAccessSparseVector").create();
   
    Option sizeOpt = obuilder.withLongName("prototypeSize").withRequired(true).withShortName("s")
        .withArgument(abuilder.withName("prototypeSize").withMinimum(1).withMaximum(1).create())
        .withDescription("The ModelDistribution prototype Vector size. ").create();
   
    Option numRedOpt = obuilder.withLongName("maxRed").withRequired(true).withShortName("r").withArgument(
      abuilder.withName("maxRed").withMinimum(1).withMaximum(1).create()).withDescription(
      "The number of reduce tasks.").create();
   
    Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt)
        .withOption(modelOpt).withOption(prototypeOpt).withOption(sizeOpt).withOption(maxIterOpt).withOption(
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.MapFile$Reader$Option

  • com.fasterxml.jackson.core.JsonLocation
  • com.hp.hpl.jena.shared.Lock
  • com.hp.hpl.jena.sparql.core.DatasetGraph
  • ivory.core.compression.BitInputStream
  • org.apache.accumulo.core.data.Key
  • org.apache.accumulo.core.data.Value
  • org.apache.accumulo.core.file.rfile.bcfile.BCFile.Reader.BlockReader
  • org.apache.accumulo.core.file.rfile.bcfile.Utils.Version
  • org.apache.avro.mapred.FsInput
  • org.apache.commons.cli.CLI2Converter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.