Examples of org.apache.hadoop.zebra.mapred.TableInputFormat

org.apache.hadoop.zebra.mapred.TableInputFormat
the tuple object for reuse. // indices of various fields in the input Tuple. int idxName, idxSalary, idxBonusPct; @Override public void configure(JobConf job) { Schema projection = TableInputFormat.getProjection(job); // determine the field indices. idxName = projection.getColumnIndex("Name"); idxSalary = projection.getColumnIndex("Salary"); idxBonusPct = projection.getColumnIndex("BonusPct"); } @Override public void map(BytesWritable key, Tuple value, OutputCollector<K, V> output, Reporter reporter) throws IOException { try { String name = (String) value.get(idxName); int salary = (Integer) value.get(idxSalary); double bonusPct = (Double) value.get(idxBonusPct); // do something with the input data } catch (ExecException e) { e.printStackTrace(); } } @Override public void close() throws IOException { // no-op } } A little bit more explanation on the PIG {@link Tuple} objects. A Tuple is anordered list of PIG datum objects. The permitted PIG datum types can be categorized as Scalar types and Composite types.
Supported Scalar types include seven native Java types: Boolean, Byte, Integer, Long, Float, Double, String, as well as one PIG class called {@link DataByteArray} that represents type-less byte array.
Supported Composite types include:
- {@link Map} : It is the same as Java Map class, with the additionalrestriction that the key-type must be one of the scalar types PIG recognizes, and the value-type any of the scaler or composite types PIG understands.
- {@link DataBag} : A DataBag is a collection of Tuples.
- {@link Tuple} : Yes, Tuple itself can be a datum in another Tuple.

    private org.apache.hadoop.zebra.schema.Schema projectionSchema;
  /**
   * default constructor
   */
  public TableLoader() {
    inputFormat = new TableInputFormat();
  }

View Full Code Here

  /**
   * @param projectionStr
   *       projection string passed from pig query.
   */
  public TableLoader(String projectionStr) {
    inputFormat = new TableInputFormat();
    projectionString = projectionStr;    
  }

View Full Code Here

   *       projection string passed from pig query.
   * @param sorted
   *      need sorted table(s)?
   */
  public TableLoader(String projectionStr, String sorted) throws IOException {
      inputFormat = new TableInputFormat();
      if (projectionStr != null && !projectionStr.isEmpty())
        projectionString = projectionStr;    
      if (sorted.equalsIgnoreCase("sorted"))
        this.sorted = true;
      else

View Full Code Here

        projection = TableInputFormat.getProjection(conf);
      } catch (ParseException e) {
        throw new IOException("Schema parsing failed :"+e.getMessage());
      }
      numProjCols = Projection.getNumColumns(projection);
      TableInputFormat inputFormat = new TableInputFormat();
      if (sorted)
        TableInputFormat.requireSortedTable(conf, null);
      scanner = inputFormat.getRecordReader(split, conf, Reporter.NULL);
      key = new BytesWritable();
    }

View Full Code Here

    BasicTable.drop(path1, conf);
    BasicTable.drop(path2, conf);
    int total1 = TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path1, true);    
    int total2 = TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path2, true);    


    TableInputFormat inputFormat = new TableInputFormat();
    TableInputFormat.setInputPaths(jobConf, path1, path2);
    TableInputFormat.setProjection(jobConf, "source_table");
    InputSplit[] splits = inputFormat.getSplits(jobConf, -1);
    Assert.assertEquals(splits.length, 2);
    for (int i = 0; i < 2; i++)
    {
      int count = 0;
      RowTableSplit split = (RowTableSplit) splits[i];
      TableRecordReader rr = (TableRecordReader) inputFormat.getRecordReader(split, jobConf, null);
      Tuple t = TypesUtils.createTuple(1);
      BytesWritable key = new BytesWritable();
      while (rr.next(key, t)) {
        int idx= (Integer) t.get(0);
        Assert.assertEquals(idx, i);

View Full Code Here

  public void testTfileSplit1() 
          throws IOException, ParseException {
    BasicTable.drop(path, conf);
    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);    


    TableInputFormat inputFormat = new TableInputFormat();
    JobConf jobConf = new JobConf(conf);
    inputFormat.setInputPaths(jobConf, path);
    inputFormat.setMinSplitSize(jobConf, 100);
    inputFormat.setProjection(jobConf, "aa");
    InputSplit[] splits = inputFormat.getSplits(jobConf, 40);
    
    RowTableSplit split = (RowTableSplit) splits[0];
    String str = split.getSplit().toString();
    StringTokenizer tokens = new StringTokenizer(str, "\n");
    str = tokens.nextToken();

View Full Code Here

    BasicTable.drop(path, conf);
    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);    
    BasicTable.dropColumnGroup(path, conf, "CG0");
    BasicTable.dropColumnGroup(path, conf, "CG2");
    
    TableInputFormat inputFormat = new TableInputFormat();
    JobConf jobConf = new JobConf(conf);
    inputFormat.setInputPaths(jobConf, path);
    inputFormat.setMinSplitSize(jobConf, 100);
    InputSplit[] splits = inputFormat.getSplits(jobConf, 40);
    
    RowTableSplit split = (RowTableSplit) splits[0];
    String str = split.getSplit().toString(); 
    StringTokenizer tokens = new StringTokenizer(str, "\n");
    str = tokens.nextToken();

View Full Code Here

    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);    
    BasicTable.dropColumnGroup(path, conf, "CG0");
    BasicTable.dropColumnGroup(path, conf, "CG1");
    BasicTable.dropColumnGroup(path, conf, "CG2");
    
    TableInputFormat inputFormat = new TableInputFormat();
    JobConf jobConf = new JobConf(conf);
    inputFormat.setInputPaths(jobConf, path);
    inputFormat.setMinSplitSize(jobConf, 100);
    InputSplit[] splits = inputFormat.getSplits(jobConf, 40);
    
    Assert.assertEquals(splits.length, 0);
  }

View Full Code Here

  
  /**
   * default constructor
   */
  public TableLoader() {
    inputFormat = new TableInputFormat();
  }

View Full Code Here

  /**
   * @param projectionStr
   *       projection string passed from pig query.
   */
  public TableLoader(String projectionStr) {
    inputFormat = new TableInputFormat();
    projectionString = projectionStr;    
  }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.zebra.mapred.TableInputFormat

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.PathFilter

org.apache.hadoop.io.BytesWritable

org.apache.hadoop.io.file.tfile.RawComparable

org.apache.hadoop.mapred.FileSplit

org.apache.hadoop.zebra.io.BasicTable.Reader.RowSplit

org.apache.hadoop.zebra.io.BasicTableStatus

org.apache.hadoop.zebra.io.BlockDistribution

org.apache.hadoop.zebra.io.KeyDistribution

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.