Examples of org.apache.hadoop.zebra.io.BasicTableStatus

org.apache.hadoop.zebra.io.BasicTableStatus
Status of a BasicTable. The status may be reported under some projection. Projection is needed because the size of a BasicTable may be affected by the projection.

    if (properClose) {
      writer = new ColumnGroup.Writer(path, conf);
      writer.close();
      /* We can only test number of rows on sorted tables.*/
      if (sorted) {
        BasicTableStatus status = getStatus(path);
        Assert.assertEquals(total, status.getRows());
      }
    }


    return total;
  }

View Full Code Here

      inserter.close();
    }


    writer = new ColumnGroup.Writer(path, conf);
    writer.close();
    BasicTableStatus status = getStatus(path);
    Assert.assertEquals(total, status.getRows());


    return total;
  }

View Full Code Here

      throw new IOException("Unable to created sorted splits");
    }


    long totalBytes = 0;
    for (Iterator<BasicTableStatus> it = status.iterator(); it.hasNext();) {
      BasicTableStatus s = it.next();
      totalBytes += s.getSize();
    }


    long maxSplits = totalBytes / getMinSplitSize(conf);


    if (maxSplits == 0)

View Full Code Here

  private static InputSplit[] getUnsortedSplits(JobConf conf, int numSplits,
      TableExpr expr, List<BasicTable.Reader> readers,
      List<BasicTableStatus> status) throws IOException {
    long totalBytes = 0;
    for (Iterator<BasicTableStatus> it = status.iterator(); it.hasNext();) {
      BasicTableStatus s = it.next();
      totalBytes += s.getSize();
    }


    long maxSplits = totalBytes / getMinSplitSize(conf);


    if (numSplits > maxSplits) {
      numSplits = -1;
    }


    ArrayList<InputSplit> ret = new ArrayList<InputSplit>();
    if (numSplits <= 0) {
      if (totalBytes <= 0) {
        BasicTable.Reader reader = readers.get(0);
        UnsortedTableSplit split =
          new UnsortedTableSplit(reader, null, conf);
        ret.add(split);
      } else {
        for (int i = 0; i < readers.size(); ++i) {
          BasicTable.Reader reader = readers.get(i);
          List<RangeSplit> subSplits = reader.rangeSplit(-1);
          for (Iterator<RangeSplit> it = subSplits.iterator(); it.hasNext();) {
            UnsortedTableSplit split =
              new UnsortedTableSplit(reader, it.next(), conf);
            ret.add(split);
          }
        }
      }
    } else {
      long goalSize = totalBytes / numSplits;


      double SPLIT_SLOP = 1.1;
      for (int i = 0; i < readers.size(); ++i) {
        BasicTable.Reader reader = readers.get(i);
        BasicTableStatus s = status.get(i);
        int nSplits =
            (int) ((s.getSize() + goalSize * (2 - SPLIT_SLOP)) / goalSize);
        if (nSplits > 1) {
          List<RangeSplit> subSplits = reader.rangeSplit(nSplits);
          for (Iterator<RangeSplit> it = subSplits.iterator(); it.hasNext();) {
            UnsortedTableSplit split =
                new UnsortedTableSplit(reader, it.next(), conf);

View Full Code Here

        BasicTable.Reader reader =
          new BasicTable.Reader(leaf.getPath(), conf);
        reader.setProjection(leaf.getProjection());
        if (sorted)
        {
          BasicTableStatus s = reader.getStatus();
          status.add(s);
        }
        readers.add(reader);
        if (first)
          first = false;

View Full Code Here

    if (properClose) {
      writer = new BasicTable.Writer(path, conf);
      writer.close();
      /* We can only test number of rows on sorted tables.*/
      if (sorted) {
        BasicTableStatus status = getStatus(path);
        Assert.assertEquals(total, status.getRows());
      }
    }


    return total;
  }

View Full Code Here

    if (properClose) {
      writer = new ColumnGroup.Writer(path, conf);
      writer.close();
      /* We can only test number of rows on sorted tables.*/
      if (sorted) {
        BasicTableStatus status = getStatus(path);
        Assert.assertEquals(total, status.getRows());
      }
    }


    return total;
  }

View Full Code Here

      inserter.close();
    }


    writer = new ColumnGroup.Writer(path, conf);
    writer.close();
    BasicTableStatus status = getStatus(path);
    Assert.assertEquals(total, status.getRows());


    return total;
  }

View Full Code Here

    if (properClose) {
      writer = new ColumnGroup.Writer(path, conf);
      writer.close();
      /* We can only test number of rows on sorted tables.*/
      if (sorted) {
        BasicTableStatus status = getStatus(path);
        Assert.assertEquals(total, status.getRows());
      }
    }


    return total;
  }

View Full Code Here

      inserter.close();
    }


    writer = new ColumnGroup.Writer(path, conf);
    writer.close();
    BasicTableStatus status = getStatus(path);
    Assert.assertEquals(total, status.getRows());


    return total;
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.zebra.io.BasicTableStatus

org.apache.hadoop.io.BytesWritable

org.apache.hadoop.zebra.io.TestBasicTable

org.apache.hadoop.zebra.io.TestColumnGroup

org.apache.hadoop.zebra.io.TestColumnGroupWithWorkPath

org.apache.hadoop.zebra.mapred.TableInputFormat

org.apache.hadoop.zebra.mapreduce.TableInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.