Examples of org.apache.hadoop.zebra.mapreduce.TableInputFormat

Package org.apache.hadoop.zebra.mapreduce

Examples of org.apache.hadoop.zebra.mapreduce.TableInputFormat

org.apache.hadoop.zebra.mapreduce.TableInputFormat
the tuple object for reuse. // indices of various fields in the input Tuple. int idxName, idxSalary, idxBonusPct; @Override public void configure(Job job) { Schema projection = TableInputFormat.getProjection(job); // determine the field indices. idxName = projection.getColumnIndex("Name"); idxSalary = projection.getColumnIndex("Salary"); idxBonusPct = projection.getColumnIndex("BonusPct"); } @Override public void map(BytesWritable key, Tuple value, OutputCollector<K, V> output, Reporter reporter) throws IOException { try { String name = (String) value.get(idxName); int salary = (Integer) value.get(idxSalary); double bonusPct = (Double) value.get(idxBonusPct); // do something with the input data } catch (ExecException e) { e.printStackTrace(); } } @Override public void close() throws IOException { // no-op } } A little bit more explanation on the PIG {@link Tuple} objects. A Tuple is anordered list of PIG datum objects. The permitted PIG datum types can be categorized as Scalar types and Composite types.
Supported Scalar types include seven native Java types: Boolean, Byte, Integer, Long, Float, Double, String, as well as one PIG class called {@link DataByteArray} that represents type-less byte array.
Supported Composite types include:
- {@link Map} : It is the same as Java Map class, with the additionalrestriction that the key-type must be one of the scalar types PIG recognizes, and the value-type any of the scaler or composite types PIG understands.
- {@link DataBag} : A DataBag is a collection of Tuples.
- {@link Tuple} : Yes, Tuple itself can be a datum in another Tuple.

  public void testTfileSplit1() 
  throws IOException, ParseException {
    BasicTable.drop(path, conf);
    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);    


    TableInputFormat inputFormat = new TableInputFormat();
    Job job = new Job(conf);
    inputFormat.setInputPaths(job, path);
    inputFormat.setMinSplitSize(job, 100);
    inputFormat.setProjection(job, "aa");
    List<InputSplit> splits = inputFormat.getSplits(job);


    RowTableSplit split = (RowTableSplit) splits.get(0);
    String str = split.getSplit().toString();
    StringTokenizer tokens = new StringTokenizer(str, "\n");
    str = tokens.nextToken();

View Full Code Here

    BasicTable.drop(path, conf);
    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);    
    BasicTable.dropColumnGroup(path, conf, "CG0");
    BasicTable.dropColumnGroup(path, conf, "CG2");


    TableInputFormat inputFormat = new TableInputFormat();
    Job job = new Job(conf);
    inputFormat.setInputPaths(job, path);
    inputFormat.setMinSplitSize(job, 100);
    List<InputSplit> splits = inputFormat.getSplits(job);


    RowTableSplit split = (RowTableSplit) splits.get( 0 );
    String str = split.getSplit().toString(); 
    StringTokenizer tokens = new StringTokenizer(str, "\n");
    str = tokens.nextToken();

View Full Code Here

    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);    
    BasicTable.dropColumnGroup(path, conf, "CG0");
    BasicTable.dropColumnGroup(path, conf, "CG1");
    BasicTable.dropColumnGroup(path, conf, "CG2");


    TableInputFormat inputFormat = new TableInputFormat();
    Job job = new Job(conf);
    inputFormat.setInputPaths(job, path);
    inputFormat.setMinSplitSize(job, 100);
    List<InputSplit> splits = inputFormat.getSplits(job);


    Assert.assertEquals(splits.size(), 0);
  }

View Full Code Here

  @Test
  public void testSortedSplitOrdering() throws IOException, ParseException {
    BasicTable.drop(path, conf);
    TestBasicTable.createBasicTable(1, 1000000, "a, b, c, d, e, f", "[a, e, d]", "a", path, true);    


    TableInputFormat inputFormat = new TableInputFormat();
    Job job = new Job(conf);
    inputFormat.setInputPaths(job, path);
    inputFormat.setMinSplitSize(job, 100);
    inputFormat.setProjection(job, "d");
    inputFormat.requireSortedTable( job, null );
    List<InputSplit> splits = inputFormat.getSplits(job);
    
    int index = 0;
    for( InputSplit is : splits ) {
      Assert.assertTrue( is instanceof SortedTableSplit );
      SortedTableSplit split = (SortedTableSplit)is;

View Full Code Here

    Job job = new Job(conf);
    System.out.println("sortKey: " + sortKey);
    TableInputFormat.setInputPaths(job, new Path(pathTable1));


    TableInputFormat.requireSortedTable(job, null);
    TableInputFormat tif = new TableInputFormat();


    SortedTableSplit split = (SortedTableSplit) tif.getSplits(job, true).get(0);


    TableScanner scanner = reader.getScanner(split.getBegin(), split.getEnd(), true);
    BytesWritable key = new BytesWritable();
    Tuple rowValue = TypesUtils.createTuple(scanner.getSchema());

View Full Code Here

     }


     @SuppressWarnings("unchecked")
     @Override
     public InputFormat getInputFormat() throws IOException {
         return new TableInputFormat();
     }

View Full Code Here

     }


     @SuppressWarnings("unchecked")
     @Override
     public InputFormat getInputFormat() throws IOException {
         return new TableInputFormat();
     }

View Full Code Here

TOP

Related Classes of org.apache.hadoop.zebra.mapreduce.TableInputFormat

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.PathFilter

org.apache.hadoop.io.BytesWritable

org.apache.hadoop.mapreduce.Job

org.apache.hadoop.mapreduce.lib.input.FileSplit

org.apache.hadoop.zebra.io.BasicTable.Reader.RowSplit

org.apache.hadoop.zebra.io.BasicTableStatus

org.apache.hadoop.zebra.io.BlockDistribution

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.