Examples of org.apache.hadoop.zebra.io.BasicTableStatus

org.apache.hadoop.zebra.io.BasicTableStatus
Status of a BasicTable. The status may be reported under some projection. Projection is needed because the size of a BasicTable may be affected by the projection.

      inserter.close();
    }


    writer = new ColumnGroup.Writer(path, conf);
    writer.close();
    BasicTableStatus status = getStatus(path);
    Assert.assertEquals(total, status.getRows());


    return total;
  }

View Full Code Here

      throw new IOException("Unable to created sorted splits");
    }


    long totalBytes = 0;
    for (Iterator<BasicTableStatus> it = status.iterator(); it.hasNext();) {
      BasicTableStatus s = it.next();
      totalBytes += s.getSize();
    }


    long maxSplits = totalBytes / getMinSplitSize(conf);


    if (numSplits > maxSplits) {

View Full Code Here

  private static InputSplit[] getUnsortedSplits(JobConf conf, int numSplits,
      TableExpr expr, List<BasicTable.Reader> readers,
      List<BasicTableStatus> status) throws IOException {
    long totalBytes = 0;
    for (Iterator<BasicTableStatus> it = status.iterator(); it.hasNext();) {
      BasicTableStatus s = it.next();
      totalBytes += s.getSize();
    }


    long maxSplits = totalBytes / getMinSplitSize(conf);


    if (numSplits > maxSplits) {
      numSplits = -1;
    }


    ArrayList<InputSplit> ret = new ArrayList<InputSplit>();;
    if (numSplits <= 0) {
      for (int i = 0; i < readers.size(); ++i) {
        BasicTable.Reader reader = readers.get(i);
        List<RangeSplit> subSplits = reader.rangeSplit(-1);
        for (Iterator<RangeSplit> it = subSplits.iterator(); it.hasNext();) {
          UnsortedTableSplit split =
              new UnsortedTableSplit(reader, it.next(), conf);
          ret.add(split);
        }
      }
    } else {
      long goalSize = totalBytes / numSplits;


      double SPLIT_SLOP = 1.1;
      for (int i = 0; i < readers.size(); ++i) {
        BasicTable.Reader reader = readers.get(i);
        BasicTableStatus s = status.get(i);
        int nSplits =
            (int) ((s.getSize() + goalSize * (2 - SPLIT_SLOP)) / goalSize);
        if (nSplits > 1) {
          List<RangeSplit> subSplits = reader.rangeSplit(nSplits);
          for (Iterator<RangeSplit> it = subSplits.iterator(); it.hasNext();) {
            UnsortedTableSplit split =
                new UnsortedTableSplit(reader, it.next(), conf);

View Full Code Here

      for (Iterator<LeafTableInfo> it = leaves.iterator(); it.hasNext();) {
        LeafTableInfo leaf = it.next();
        BasicTable.Reader reader =
          new BasicTable.Reader(leaf.getPath(), conf);
        reader.setProjection(leaf.getProjection());
        BasicTableStatus s = reader.getStatus();
        if ((s.getRows() != 0) && (s.getSize() > 0)) {
          // skipping empty tables.
          // getRows() may return -1 to indicate unknown row numbers.
          readers.add(reader);
          status.add(s);
        } else {

View Full Code Here

    }


    if (properClose) {
      writer = new ColumnGroup.Writer(path, conf);
      writer.close();
      BasicTableStatus status = getStatus(path);
      Assert.assertEquals(total, status.getRows());
    }


    return total;
  }

View Full Code Here

      inserter.close();
    }


    writer = new ColumnGroup.Writer(path, conf);
    writer.close();
    BasicTableStatus status = getStatus(path);
    Assert.assertEquals(total, status.getRows());


    return total;
  }

View Full Code Here

    }


    if (properClose) {
      writer = new BasicTable.Writer(path, conf);
      writer.close();
      BasicTableStatus status = getStatus(path);
      Assert.assertEquals(total, status.getRows());
    }


    return total;
  }

View Full Code Here

    if (properClose) {
      writer = new BasicTable.Writer(path, conf);
      writer.close();
      /* We can only test number of rows on sorted tables.*/
      if (sorted) {
        BasicTableStatus status = getStatus(path);
        Assert.assertEquals(total, status.getRows());
      }
    }


    return total;
  }

View Full Code Here

      throw new IOException("Unable to created sorted splits");
    }


    long totalBytes = 0;
    for (Iterator<BasicTableStatus> it = status.iterator(); it.hasNext();) {
      BasicTableStatus s = it.next();
      totalBytes += s.getSize();
    }


    long maxSplits = totalBytes / getMinSplitSize( conf );
    List<InputSplit> splits = new ArrayList<InputSplit>();
    if (maxSplits == 0)

View Full Code Here

        for (Iterator<LeafTableInfo> it = leaves.iterator(); it.hasNext();) {
          LeafTableInfo leaf = it.next();
          BasicTable.Reader reader =
            new BasicTable.Reader(leaf.getPath(), conf );
          reader.setProjection(leaf.getProjection());
          BasicTableStatus s = reader.getStatus();
            status.add(s);
          readers.add(reader);
        }


        if( readers.isEmpty() ) {

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.zebra.io.BasicTableStatus

org.apache.hadoop.io.BytesWritable

org.apache.hadoop.zebra.io.TestBasicTable

org.apache.hadoop.zebra.io.TestColumnGroup

org.apache.hadoop.zebra.io.TestColumnGroupWithWorkPath

org.apache.hadoop.zebra.mapred.TableInputFormat

org.apache.hadoop.zebra.mapreduce.TableInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.