Examples of HBaseSourceTarget


Examples of org.apache.crunch.io.hbase.HBaseSourceTarget

    Scan scan = new Scan();
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_PLAY);
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_QUOTE);

    // Our hbase source
    HBaseSourceTarget source = new HBaseSourceTarget(TABLE_SOURCE, scan);

    // Our source, in a format which can be use by crunch
    PTable<ImmutableBytesWritable, Result> rawText = pipeline.read(source);

    // We process the data from the source HTable then concatenate all data
View Full Code Here

Examples of org.apache.crunch.io.hbase.HBaseSourceTarget

      key = put(inputTable, key, "cat");
      key = put(inputTable, key, "cat");
      key = put(inputTable, key, "dog");
      Scan scan = new Scan();
      scan.addFamily(WORD_COLFAM);
      HBaseSourceTarget source = new HBaseSourceTarget(inputTableName, scan);
      PTable<ImmutableBytesWritable, Result> words = pipeline.read(source);
      PCollection<Put> puts = wordCount(words);
      pipeline.write(puts, new HBaseTarget(outputTableName));
      pipeline.write(puts, new HBaseTarget(otherTableName));
      pipeline.done();

      assertIsLong(outputTable, "cat", 2);
      assertIsLong(outputTable, "dog", 1);
      assertIsLong(otherTable, "cat", 2);
      assertIsLong(otherTable, "dog", 1);
     
      // verify we can do joins.
      HTable joinTable = hbaseTestUtil.createTable(Bytes.toBytes(joinTableName), WORD_COLFAM);
      key = 0;
      key = put(joinTable, key, "zebra");
      key = put(joinTable, key, "donkey");
      key = put(joinTable, key, "bird");
      key = put(joinTable, key, "horse");
     
      Scan joinScan = new Scan();
      joinScan.addFamily(WORD_COLFAM);
      PTable<ImmutableBytesWritable, Result> other = pipeline.read(FromHBase.table(joinTableName, joinScan));
      PCollection<String> joined = words.join(other).parallelDo(new StringifyFn(), Writables.strings());
      assertEquals(ImmutableSet.of("cat,zebra", "cat,donkey", "dog,bird"),
          ImmutableSet.copyOf(joined.materialize()));
      pipeline.done();

      //verify HBaseTarget supports deletes.
      Scan clearScan = new Scan();
      clearScan.addFamily(COUNTS_COLFAM);
      pipeline = new MRPipeline(WordCountHBaseIT.class, hbaseTestUtil.getConfiguration());
      HBaseSourceTarget clearSource = new HBaseSourceTarget(outputTableName, clearScan);
      PTable<ImmutableBytesWritable, Result> counts = pipeline.read(clearSource);
      pipeline.write(clearCounts(counts), new HBaseTarget(outputTableName));
      pipeline.done();
     
      assertDeleted(outputTable, "cat");
View Full Code Here

Examples of org.apache.crunch.io.hbase.HBaseSourceTarget

    Scan scan = new Scan();
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_PLAY);
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_QUOTE);

    // Our hbase source
    HBaseSourceTarget source = new HBaseSourceTarget(TABLE_SOURCE, scan);

    // Our source, in a format which can be use by crunch
    PTable<ImmutableBytesWritable, Result> rawText = pipeline.read(source);

    // We process the data from the source HTable then concatenate all data
View Full Code Here

Examples of org.apache.crunch.io.hbase.HBaseSourceTarget

    Scan scan = new Scan();
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_PLAY);
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_QUOTE);

    // Our hbase source
    HBaseSourceTarget source = new HBaseSourceTarget(TABLE_SOURCE, scan);

    // Our source, in a format which can be use by crunch
    PTable<ImmutableBytesWritable, Result> rawText = pipeline.read(source);

    // We process the data from the source HTable then concatenate all data
View Full Code Here

Examples of org.apache.crunch.io.hbase.HBaseSourceTarget

    Scan scan = new Scan();
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_PLAY);
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_QUOTE);

    // Our hbase source
    HBaseSourceTarget source = new HBaseSourceTarget(TABLE_SOURCE, scan);

    // Our source, in a format which can be use by crunch
    PTable<ImmutableBytesWritable, Result> rawText = pipeline.read(source);

    // We process the data from the source HTable then concatenate all data
View Full Code Here

Examples of org.apache.crunch.io.hbase.HBaseSourceTarget

    Scan scan = new Scan();
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_PLAY);
    scan.addColumn(COLUMN_FAMILY_SOURCE, COLUMN_QUALIFIER_SOURCE_QUOTE);

    // Our hbase source
    HBaseSourceTarget source = new HBaseSourceTarget(TABLE_SOURCE, scan);

    // Our source, in a format which can be use by crunch
    PTable<ImmutableBytesWritable, Result> rawText = pipeline.read(source);

    // We process the data from the source HTable then concatenate all data
View Full Code Here

Examples of org.apache.crunch.io.hbase.HBaseSourceTarget

      key = put(inputTable, key, "cat");
      key = put(inputTable, key, "cat");
      key = put(inputTable, key, "dog");
      Scan scan = new Scan();
      scan.addColumn(WORD_COLFAM, null);
      HBaseSourceTarget source = new HBaseSourceTarget(inputTableName, scan);
      PTable<ImmutableBytesWritable, Result> words = pipeline.read(source);
      PCollection<Put> puts = wordCount(words);
      pipeline.write(puts, new HBaseTarget(outputTableName));
      pipeline.write(puts, new HBaseTarget(otherTableName));
      pipeline.done();

      assertIsLong(outputTable, "cat", 2);
      assertIsLong(outputTable, "dog", 1);
      assertIsLong(otherTable, "cat", 2);
      assertIsLong(otherTable, "dog", 1);
     
      // verify we can do joins.
      HTable joinTable = hbaseTestUtil.createTable(Bytes.toBytes(joinTableName), WORD_COLFAM);
      key = 0;
      key = put(joinTable, key, "zebra");
      key = put(joinTable, key, "donkey");
      key = put(joinTable, key, "bird");
      key = put(joinTable, key, "horse");
     
      Scan joinScan = new Scan();
      joinScan.addColumn(WORD_COLFAM, null);
      PTable<ImmutableBytesWritable, Result> other = pipeline.read(FromHBase.table(joinTableName, joinScan));
      PCollection<String> joined = words.join(other).parallelDo(new StringifyFn(), Writables.strings());
      assertEquals(ImmutableSet.of("cat,zebra", "cat,donkey", "dog,bird"),
          ImmutableSet.copyOf(joined.materialize()));
      pipeline.done();

      //verify HBaseTarget supports deletes.
      Scan clearScan = new Scan();
      clearScan.addColumn(COUNTS_COLFAM, null);
      pipeline = new MRPipeline(WordCountHBaseIT.class, hbaseTestUtil.getConfiguration());
      HBaseSourceTarget clearSource = new HBaseSourceTarget(outputTableName, clearScan);
      PTable<ImmutableBytesWritable, Result> counts = pipeline.read(clearSource);
      pipeline.write(clearCounts(counts), new HBaseTarget(outputTableName));
      pipeline.done();
     
      assertDeleted(outputTable, "cat");
View Full Code Here

Examples of org.apache.crunch.io.hbase.HBaseSourceTarget

      key = put(inputTable, key, "cat");
      key = put(inputTable, key, "cat");
      key = put(inputTable, key, "dog");
      Scan scan = new Scan();
      scan.addColumn(WORD_COLFAM, null);
      HBaseSourceTarget source = new HBaseSourceTarget(inputTableName, scan);
      PTable<ImmutableBytesWritable, Result> shakespeare = pipeline.read(source);
      pipeline.write(wordCount(shakespeare), new HBaseTarget(outputTableName));
      pipeline.done();

      assertIsLong(outputTable, "cat", 2);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.