Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.NullWritable


    assertEquals(1, splits.length);

    // read the whole file
    org.apache.hadoop.mapred.RecordReader<NullWritable, OrcLazyRow> reader =
        in.getRecordReader(splits[0], conf, Reporter.NULL);
    NullWritable key = reader.createKey();
    OrcLazyRow value = (OrcLazyRow) reader.createValue();
    List<? extends StructField> fields =inspector.getAllStructFieldRefs();
    StringObjectInspector strInspector = (StringObjectInspector)
        fields.get(0).getFieldObjectInspector();
    assertEquals(true, reader.next(key, value));
View Full Code Here


     * @throws IOException 読み込みに失敗した場合
     * @deprecated Use {@link #load(ModelInput)} instead
     */
    @Deprecated
    public void load(SequenceFile.Reader reader) throws IOException {
        NullWritable key = NullWritable.get();
        for (;;) {
            Writable model;
            try {
                model = modelClass.newInstance();
            } catch (InstantiationException e) {
View Full Code Here

  public static Vector retrieveTimesSquaredOutputVector(JobConf conf) throws IOException {
    Path outputPath = FileOutputFormat.getOutputPath(conf);
    FileSystem fs = FileSystem.get(conf);
    Path outputFile = new Path(outputPath, "part-00000");
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, outputFile, conf);
    NullWritable n = NullWritable.get();
    VectorWritable v = new VectorWritable();
    reader.next(n,v);
    Vector vector = v.get();
    reader.close();
    fs.deleteOnExit(outputFile);
View Full Code Here

        SequenceFile.Reader reader = new SequenceFile.Reader(fs,
          inputVectorPath,
          conf);
        VectorWritable val = new VectorWritable();
        NullWritable nw = NullWritable.get();
        reader.next(nw, val);
        reader.close();
        inputVector = val.get();
        if(!(inputVector instanceof SequentialAccessSparseVector || inputVector instanceof DenseVector)) {
          inputVector = new SequentialAccessSparseVector(inputVector);
View Full Code Here

        InMemRecordReader reader = (InMemRecordReader) inputFormat.getRecordReader(
            split, conf, null);

        for (int tree = 0; tree < split.getNbTrees(); tree++) {
          IntWritable key = reader.createKey();
          NullWritable value = reader.createValue();

          // reader.next() should return true until there is no tree left
          assertEquals(tree < split.getNbTrees(), reader.next(key, value));
         
          assertEquals(split.getFirstId() + tree, key.get());
View Full Code Here

    if (fs.exists(dst)) {
      fs.delete(dst, false);
    }
    SequenceFile.Writer writer = SequenceFile.createWriter(fs,
      conf, dst, job.getMapOutputKeyClass(), NullWritable.class);
    NullWritable nullValue = NullWritable.get();
    float stepSize = samples.length / (float) numPartitions;
    int last = -1;
    for(int i = 1; i < numPartitions; ++i) {
      int k = Math.round(stepSize * i);
      while (last >= k && comparator.compare(samples[last], samples[k]) == 0) {
View Full Code Here

    Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
        "totalordermemcmp", conf, splitStrings);
    try {
      partitioner.setConf(conf);
      NullWritable nw = NullWritable.get();
      for (Check<Text> chk : testStrings) {
        assertEquals(chk.data.toString(), chk.part,
            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
      }
    } finally {
View Full Code Here

    Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
        "totalorderbinarysearch", conf, splitStrings);
    conf.setBoolean(TotalOrderPartitioner.NATURAL_ORDER, false);
    try {
      partitioner.setConf(conf);
      NullWritable nw = NullWritable.get();
      for (Check<Text> chk : testStrings) {
        assertEquals(chk.data.toString(), chk.part,
            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
      }
    } finally {
View Full Code Here

    revCheck.add(new Check<Text>(new Text("z"), 0));
    revCheck.add(new Check<Text>(new Text("ddngo"), 4));
    revCheck.add(new Check<Text>(new Text("hi"), 3));
    try {
      partitioner.setConf(conf);
      NullWritable nw = NullWritable.get();
      for (Check<Text> chk : revCheck) {
        assertEquals(chk.data.toString(), chk.part,
            partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
      }
    } finally {
View Full Code Here

  private Object[] readPartitions(FileSystem fs, Path p, Class<?> keyClass,
      Configuration conf) throws IOException {
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
    ArrayList<Object> parts = new ArrayList<Object>();
    Writable key = (Writable)ReflectionUtils.newInstance(keyClass, conf);
    NullWritable value = NullWritable.get();
    while (reader.next(key, value)) {
      parts.add(key);
      key = (Writable)ReflectionUtils.newInstance(keyClass, conf);
    }
    reader.close();
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.NullWritable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.