Examples of FileFragment

Examples of org.apache.tajo.storage.fragment.FileFragment

40
41
42
43
44
45
46
47
48
49
50
51
    path = CommonTestingUtil.getTestDir();
  }

  @Test
  public final void testGetAndSetFields() {
    FileFragment fragment1 = new FileFragment("table1_1", new Path(path, "table0"), 0, 500);

    assertEquals("table1_1", fragment1.getTableName());
    assertEquals(new Path(path, "table0"), fragment1.getPath());
    assertTrue(0 == fragment1.getStartKey());
    assertTrue(500 == fragment1.getEndKey());
  }
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

50
51
52
53
54
55
56
57
58
59
60
61
62
    assertTrue(500 == fragment1.getEndKey());
  }

  @Test
  public final void testGetProtoAndRestore() {
    FileFragment fragment = new FileFragment("table1_1", new Path(path, "table0"), 0, 500);

    FileFragment fragment1 = FragmentConvertor.convert(FileFragment.class, fragment.getProto());
    assertEquals("table1_1", fragment1.getTableName());
    assertEquals(new Path(path, "table0"), fragment1.getPath());
    assertTrue(0 == fragment1.getStartKey());
    assertTrue(500 == fragment1.getEndKey());
  }
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

64
65
66
67
68
69
70
71
72
73
74
  @Test
  public final void testCompareTo() {
    final int num = 10;
    FileFragment[] tablets = new FileFragment[num];
    for (int i = num - 1; i >= 0; i--) {
      tablets[i] = new FileFragment("tablet1_"+i, new Path(path, "tablet0"), i * 500, (i+1) * 500);
    }
   
    Arrays.sort(tablets);

    for(int i = 0; i < num; i++) {
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

79
80
81
82
83
84
85
86
87
88
89
  @Test
  public final void testCompareTo2() {
    final int num = 1860;
    FileFragment[] tablets = new FileFragment[num];
    for (int i = num - 1; i >= 0; i--) {
      tablets[i] = new FileFragment("tablet1_"+i, new Path(path, "tablet0"), (long)i * 6553500, (long)(i+1) * 6553500);
    }

    SortedSet sortedSet = Sets.newTreeSet();
    for (FileFragment frag : tablets) {
      sortedSet.add(frag);
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

126
127
128
129
130
131
132
133
134
                          final CatalogProtos.FragmentProto[] fragments) throws PhysicalPlanningException {
    this(context, sm, plan);

    mergedInputPaths = TUtil.newList();
    for (CatalogProtos.FragmentProto proto : fragments) {
      FileFragment fragment = FragmentConvertor.convert(FileFragment.class, proto);
      mergedInputPaths.add(fragment.getPath());
    }
  }
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
        childBlocks[0] = masterPlan.getChild(execBlock.getId(), 0);
        childBlocks[1] = masterPlan.getChild(execBlock.getId(), 1);

        tablePath = storageManager.getTablePath(scans[i].getTableName());
        stats[i] = masterContext.getSubQuery(childBlocks[i].getId()).getResultStats().getNumBytes();
        fragments[i] = new FileFragment(scans[i].getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST});
      } else {
        tablePath = tableDesc.getPath();
        try {
          stats[i] = GlobalPlanner.computeDescendentVolume(scans[i]);
        } catch (PlanningException e) {
          throw new IOException(e);
        }

        // if table has no data, storageManager will return empty FileFragment.
        // So, we need to handle FileFragment by its size.
        // If we don't check its size, it can cause IndexOutOfBoundsException.
        List<FileFragment> fileFragments = storageManager.getSplits(scans[i].getCanonicalName(), tableDesc.getMeta(), tableDesc.getSchema(), tablePath);
        if (fileFragments.size() > 0) {
          fragments[i] = fileFragments.get(0);
        } else {
          fragments[i] = new FileFragment(scans[i].getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST});
        }
      }
    }

    // If one of inner join tables has no input data,
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

383
384
385
386
387
388
389
390
391
392
393
    LOG.info("Try to divide " + mergedRange + " into " + determinedTaskNum +
        " sub ranges (total units: " + determinedTaskNum + ")");
    TupleRange [] ranges = partitioner.partition(determinedTaskNum);

    FileFragment dummyFragment = new FileFragment(scan.getTableName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST});
    SubQuery.scheduleFragment(subQuery, dummyFragment);

    List<String> basicFetchURIs = new ArrayList<String>();
    List<ExecutionBlock> childBlocks = masterPlan.getChilds(subQuery.getId());
    for (ExecutionBlock childBlock : childBlocks) {
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

476
477
478
479
480
481
482
483
484
485
486
    ScanNode scan = execBlock.getScanNodes()[0];
    Path tablePath;
    tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName());

    FileFragment frag = new FileFragment(scan.getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST});
    List<FileFragment> fragments = new ArrayList<FileFragment>();
    fragments.add(frag);
    SubQuery.scheduleFragments(subQuery, fragments);

    Map<String, List<IntermediateEntry>> hashedByHost;
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
    c.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "file:///");
    FileSystem fs = FileSystem.get(c);
    Path tablePath = new Path(file.getAbsolutePath());

    List<FileFragment> listTablets = new ArrayList<FileFragment>();
    FileFragment tablet;

    FileStatus[] fileLists = fs.listStatus(tablePath);
    for (FileStatus f : fileLists) {
      if (f.getLen() == 0) {
        continue;
      }
      tablet = new FileFragment(name, f.getPath(), 0l, f.getLen());
      listTablets.add(tablet);
    }

    FileFragment[] tablets = new FileFragment[listTablets.size()];
    listTablets.toArray(tablets);
View Full Code Here

Examples of org.apache.tajo.storage.fragment.FileFragment

105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
    FileStatus file = fs.getFileStatus(dataPath);
    TableProto proto = (TableProto) FileUtil.loadProto(
        cluster.getDefaultFileSystem(), metaPath, TableProto.getDefaultInstance());
    meta = new TableMeta(proto);
    FileFragment fragment = new FileFragment("test.tbl", dataPath, 0, file.getLen());

    int tupleCnt = 0;
    start = System.currentTimeMillis();
    Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
    scanner.init();
    while ((tuple=scanner.next()) != null) {
      tupleCnt++;
    }
    scanner.close();

    assertEquals(tupleNum, tupleCnt);

    tupleCnt = 0;
    long fileStart = 0;
    long fileLen = file.getLen()/13;

    for (int i = 0; i < 13; i++) {
      fragment = new FileFragment("test.tbl", dataPath, fileStart, fileLen);
      scanner = new RowFile.RowFileScanner(conf, schema, meta, fragment);
      scanner.init();
      while ((tuple=scanner.next()) != null) {
        if (!idSet.remove(tuple.get(0).asInt4())) {
          System.out.println("duplicated! " + tuple.get(0).asInt4());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.