Package org.apache.tajo.engine.planner

Examples of org.apache.tajo.engine.planner.UniformRangePartition

323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
    SortSpec [] sortSpecs = sortNode.getSortKeys();
    Schema sortSchema = new Schema(channel.getPartitionKey());

    // calculate the number of maximum query ranges
    TupleRange mergedRange = TupleUtil.columnStatToRange(channel.getSchema(), sortSchema, stat.getColumnStats());
    RangePartitionAlgorithm partitioner = new UniformRangePartition(sortSchema, mergedRange);
    BigDecimal card = partitioner.getTotalCardinality();

    // if the number of the range cardinality is less than the desired number of tasks,
    // we set the the number of tasks to the number of range cardinality.
    int determinedTaskNum;
    if (card.compareTo(new BigDecimal(maxNum)) < 0) {
      LOG.info("The range cardinality (" + card
          + ") is less then the desired number of tasks (" + maxNum + ")");
      determinedTaskNum = card.intValue();
    } else {
      determinedTaskNum = maxNum;
    }

    LOG.info("Try to divide " + mergedRange + " into " + determinedTaskNum +
        " sub ranges (total units: " + determinedTaskNum + ")");
    TupleRange [] ranges = partitioner.partition(determinedTaskNum);

    Fragment dummyFragment = new Fragment(scan.getTableName(), tablePath,
        CatalogUtil.newTableMeta(scan.getInSchema(), StoreType.CSV),0, 0);

    List<String> basicFetchURIs = new ArrayList<String>();
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
    eTuple.put(3, DatumFactory.createInt4(70));
    eTuple.put(4, DatumFactory.createInt8(10000));
    eTuple.put(5, DatumFactory.createFloat4(150));
    eTuple.put(6, DatumFactory.createFloat8(170));

    RangePartitionAlgorithm partitioner = new UniformRangePartition(schema, new TupleRange(schema, sTuple, eTuple));
    TupleRange [] ranges = partitioner.partition(5);
    assertTrue(5 <= ranges.length);
    TupleComparator comp = new TupleComparator(schema, PlannerUtil.schemaToSortSpecs(schema));
    TupleRange prev = ranges[0];
    for (int i = 1; i < ranges.length; i++) {
      assertTrue(comp.compare(prev.getStart(), ranges[i].getStart()) < 0);
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
227
    Tuple e = new VTuple(2);
    e.put(0, DatumFactory.createText("R"));
    e.put(1, DatumFactory.createText("O"));
    TupleRange expected = new TupleRange(schema, s, e);

    RangePartitionAlgorithm partitioner = new UniformRangePartition(schema, expected, true);
    TupleRange [] ranges = partitioner.partition(31);

    String query;
    for (TupleRange range : ranges) {
      query = TupleUtil.rangeToQuery(schema, range, true, false);
      TupleRange result = TupleUtil.queryToRange(schema, query);
View Full Code Here
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
    // If there is an empty table in inner join, it should return zero rows.
    if (totalStat.getNumBytes() == 0 && totalStat.getColumnStats().size() == 0 ) {
      return;
    }
    TupleRange mergedRange = TupleUtil.columnStatToRange(sortSpecs, sortSchema, totalStat.getColumnStats());
    RangePartitionAlgorithm partitioner = new UniformRangePartition(mergedRange, sortSpecs);
    BigDecimal card = partitioner.getTotalCardinality();

    // if the number of the range cardinality is less than the desired number of tasks,
    // we set the the number of tasks to the number of range cardinality.
    int determinedTaskNum;
    if (card.compareTo(new BigDecimal(maxNum)) < 0) {
      LOG.info("The range cardinality (" + card
          + ") is less then the desired number of tasks (" + maxNum + ")");
      determinedTaskNum = card.intValue();
    } else {
      determinedTaskNum = maxNum;
    }

    LOG.info("Try to divide " + mergedRange + " into " + determinedTaskNum +
        " sub ranges (total units: " + determinedTaskNum + ")");
    TupleRange [] ranges = partitioner.partition(determinedTaskNum);

    FileFragment dummyFragment = new FileFragment(scan.getTableName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST});
    SubQuery.scheduleFragment(subQuery, dummyFragment);

    List<String> basicFetchURIs = new ArrayList<String>();
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
    eTuple.put(3, DatumFactory.createInt4(70));
    eTuple.put(4, DatumFactory.createInt8(10000));
    eTuple.put(5, DatumFactory.createFloat4(150));
    eTuple.put(6, DatumFactory.createFloat8(170));

    RangePartitionAlgorithm partitioner = new UniformRangePartition(new TupleRange(sortSpecs, sTuple, eTuple),
        sortSpecs);
    TupleRange [] ranges = partitioner.partition(5);
    assertTrue(5 <= ranges.length);
    TupleComparator comp = new TupleComparator(schema, PlannerUtil.schemaToSortSpecs(schema));
    TupleRange prev = ranges[0];
    for (int i = 1; i < ranges.length; i++) {
      assertTrue(comp.compare(prev.getStart(), ranges[i].getStart()) < 0);
View Full Code Here
TOP

Related Classes of org.apache.tajo.engine.planner.UniformRangePartition

  • org.apache.tajo.catalog.Column
  • org.apache.tajo.engine.util.TestTupleUtil
  • org.apache.tajo.master.querymaster.Repartitioner
  • org.apache.tajo.storage.Tuple
  • org.apache.tajo.storage.TupleRange
  • org.apache.tajo.storage.VTuple
  • org.apache.tajo.engine.exception.RangeOverflowException
  • java.math.BigDecimal
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.