Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.Reader


      }
      final int COL_COUNT = 10;
      final Text regionName = new Text("regionname");
      final Text tableName = new Text("tablename");
      final Text row = new Text("row");
      Reader reader = null;
      HLog log = new HLog(fs, dir, this.conf);
      try {
        // Write columns named 1, 2, 3, etc. and then values of single byte
        // 1, 2, 3...
        TreeMap<Text, byte []> cols = new TreeMap<Text, byte []>();
        for (int i = 0; i < COL_COUNT; i++) {
          cols.put(new Text(Integer.toString(i)),
            new byte[] { (byte)(i + '0') });
        }
        long timestamp = System.currentTimeMillis();
        log.append(regionName, tableName, row, cols, timestamp);
        long logSeqId = log.startCacheFlush();
        log.completeCacheFlush(regionName, tableName, logSeqId);
        log.close();
        Path filename = log.computeFilename(log.filenum - 1);
        log = null;
        // Now open a reader on the log and assert append worked.
        reader = new SequenceFile.Reader(fs, filename, conf);
        HLogKey key = new HLogKey();
        HLogEdit val = new HLogEdit();
        for (int i = 0; i < COL_COUNT; i++) {
          reader.next(key, val);
          assertEquals(regionName, key.getRegionName());
          assertEquals(tableName, key.getTablename());
          assertEquals(row, key.getRow());
          assertEquals((byte)(i + '0'), val.getVal()[0]);
          System.out.println(key + " " + val);
        }
        while (reader.next(key, val)) {
          // Assert only one more row... the meta flushed row.
          assertEquals(regionName, key.getRegionName());
          assertEquals(tableName, key.getTablename());
          assertEquals(HLog.METAROW, key.getRow());
          assertEquals(HLog.METACOLUMN, val.getColumn());
          assertEquals(0, COMPLETE_CACHEFLUSH.compareTo(val.getVal()));
          System.out.println(key + " " + val);
        }
      } finally {
        if (log != null) {
          log.closeAndDelete();
        }
        if (reader != null) {
          reader.close();
        }
        if (fs.exists(dir)) {
          fs.delete(dir);
        }
      }
View Full Code Here


      throw new FileNotFoundException(args[0] + " does not exist");
    }
    if (!fs.isFile(logfile)) {
      throw new IOException(args[0] + " is not a file");
    }
    Reader log = new SequenceFile.Reader(fs, logfile, conf);
    try {
      HLogKey key = new HLogKey();
      HLogEdit val = new HLogEdit();
      while(log.next(key, val)) {
        System.out.println(key.toString() + " " + val.toString());
      }
    } finally {
      log.close();
    }
  }
View Full Code Here

  public void testAppend() throws IOException {
    final int COL_COUNT = 10;
    final Text regionName = new Text("regionname");
    final Text tableName = new Text("tablename");
    final Text row = new Text("row");
    Reader reader = null;
    HLog log = new HLog(fs, dir, this.conf, null);
    try {
      // Write columns named 1, 2, 3, etc. and then values of single byte
      // 1, 2, 3...
      long timestamp = System.currentTimeMillis();
      TreeMap<HStoreKey, byte []> cols = new TreeMap<HStoreKey, byte []>();
      for (int i = 0; i < COL_COUNT; i++) {
        cols.put(new HStoreKey(row, new Text(Integer.toString(i)), timestamp),
            new byte[] { (byte)(i + '0') });
      }
      log.append(regionName, tableName, cols);
      long logSeqId = log.startCacheFlush();
      log.completeCacheFlush(regionName, tableName, logSeqId);
      log.close();
      Path filename = log.computeFilename(log.filenum - 1);
      log = null;
      // Now open a reader on the log and assert append worked.
      reader = new SequenceFile.Reader(fs, filename, conf);
      HLogKey key = new HLogKey();
      HLogEdit val = new HLogEdit();
      for (int i = 0; i < COL_COUNT; i++) {
        reader.next(key, val);
        assertEquals(regionName, key.getRegionName());
        assertEquals(tableName, key.getTablename());
        assertEquals(row, key.getRow());
        assertEquals((byte)(i + '0'), val.getVal()[0]);
        System.out.println(key + " " + val);
      }
      while (reader.next(key, val)) {
        // Assert only one more row... the meta flushed row.
        assertEquals(regionName, key.getRegionName());
        assertEquals(tableName, key.getTablename());
        assertEquals(HLog.METAROW, key.getRow());
        assertEquals(HLog.METACOLUMN, val.getColumn());
        assertEquals(0, HLogEdit.completeCacheFlush.compareTo(val.getVal()));
        System.out.println(key + " " + val);
      }
    } finally {
      if (log != null) {
        log.closeAndDelete();
      }
      if (reader != null) {
        reader.close();
      }
    }
  }
View Full Code Here

      }
      if (dump) {
        if (!fs.isFile(logPath)) {
          throw new IOException(args[i] + " is not a file");
        }
        Reader log = new SequenceFile.Reader(fs, logPath, conf);
        try {
          HLogKey key = new HLogKey();
          HLogEdit val = new HLogEdit();
          while (log.next(key, val)) {
            System.out.println(key.toString() + " " + val.toString());
          }
        } finally {
          log.close();
        }
      } else {
        if (!fs.getFileStatus(logPath).isDir()) {
          throw new IOException(args[i] + " is not a directory");
        }
View Full Code Here

    sorter.merge(outfiles, output);

    // import the evaluations
    LongWritable key = new LongWritable();
    DoubleWritable value = new DoubleWritable();
    Reader reader = new Reader(fs, output, conf);
    try {
      while (reader.next(key, value)) {
        evaluations.add(value.get());
      }
    } finally {
      reader.close();
    }
  }
View Full Code Here

    sorter.merge(outfiles, output);

    // import the evaluations
    LongWritable key = new LongWritable();
    CDFitness value = new CDFitness();
    Reader reader = new Reader(fs, output, conf);

    while (reader.next(key, value)) {
      evaluations.add(new CDFitness(value));
    }

    reader.close();
  }
View Full Code Here

    sorter.merge(outfiles, output);

    // import the descriptions
    LongWritable key = new LongWritable();
    Text value = new Text();
    Reader reader = new Reader(fs, output, conf);

    while (reader.next(key, value)) {
      descriptions.add(value.toString());
    }

    reader.close();
  }
View Full Code Here

    PCollection<String> collection = MemPipeline.typedCollectionOf(Writables.strings(), EXPECTED_COLLECTION);
    final Target target = To.sequenceFile(outputDir.toString());
    MemPipeline.getInstance().write(collection, target);

    // read
    final SequenceFile.Reader reader = new Reader(FileSystem.getLocal(
      baseTmpDir.getDefaultConfiguration()),
        new Path(getOutputFile(outputDir, "*.seq").toString()),
        baseTmpDir.getDefaultConfiguration());
    final List<String> actual = Lists.newArrayList();
    final NullWritable key = NullWritable.get();
    final Text value = new Text();
    while (reader.next(key, value)) {
      actual.add(value.toString());
    }
    reader.close();

    // assert read same as written
    assertEquals(EXPECTED_COLLECTION, actual);
  }
View Full Code Here

          Writables.strings()), "test input");
    final Target target = To.sequenceFile(outputDir.toString());
    MemPipeline.getInstance().write(collection, target);

    // read
    final SequenceFile.Reader reader = new Reader(FileSystem.getLocal(baseTmpDir
        .getDefaultConfiguration()), new Path(getOutputFile(outputDir, "*.seq").toString()),
        baseTmpDir.getDefaultConfiguration());
    final List<Pair<Integer, String>> actual = Lists.newArrayList();
    final IntWritable key = new IntWritable();
    final Text value = new Text();
    while (reader.next(key, value)) {
      actual.add(Pair.of(key.get(), value.toString()));
    }
    reader.close();

    // assert read same as written
    assertEquals(EXPECTED_TABLE, actual);
  }
View Full Code Here

      } else {
        classifier = new StandardNaiveBayesClassifier(model);
      }
      SequenceFile.Writer writer =
          new SequenceFile.Writer(fs, getConf(), getOutputPath(), Text.class, VectorWritable.class);
      Reader reader = new Reader(fs, getInputPath(), getConf());
      Text key = new Text();
      VectorWritable vw = new VectorWritable();
      while (reader.next(key, vw)) {
        writer.append(new Text(SLASH.split(key.toString())[1]),
            new VectorWritable(classifier.classifyFull(vw.get())));
      }
      writer.close();
      reader.close();
    } else {
      boolean succeeded = runMapReduce(parsedArgs);
      if (!succeeded) {
        return -1;
      }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Reader

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.