Package org.apache.hadoop.zebra.types

Examples of org.apache.hadoop.zebra.types.CGSchema

187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
      System.out.println("********** Column Groups **********");
      for (int i = 0; i < cgschemas.length; i++) {
        System.out.println(cgschemas[i]);
        System.out.println("--------------------------------");
      }
      CGSchema cgs1 = cgschemas[0];
      CGSchema cgs2 = cgschemas[1];
      CGSchema cgs3 = cgschemas[2];

      ColumnSchema f11 = cgs1.getSchema().getColumn(0);
      Assert.assertEquals("c.r.f1", f11.name);
      Assert.assertEquals(ColumnType.INT, f11.type);

      ColumnSchema f21 = cgs2.getSchema().getColumn(0);
      Assert.assertEquals("m1", f21.name);
      Assert.assertEquals(ColumnType.MAP, f21.type);

      ColumnSchema f31 = cgs3.getSchema().getColumn(0);
      Assert.assertEquals("c.r.f2", f31.name);
      Assert.assertEquals(ColumnType.INT, f31.type);

      Assert.assertEquals(cgs1.getCompressor(), "gzip");
      Assert.assertEquals(cgs1.getSerializer(), "pig");
      Assert.assertEquals(cgs2.getCompressor(), "lzo2");
      Assert.assertEquals(cgs2.getSerializer(), "avro");
      Assert.assertEquals(cgs3.getCompressor(), "lzo2");
      Assert.assertEquals(cgs3.getSerializer(), "pig");

      System.out.println("*********** Column Map **********");
      Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
          .getPartitionInfo().getColMap();
      Assert.assertEquals(colmap.size(), 3);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
      System.out.println("********** Column Groups **********");
      for (int i = 0; i < cgschemas.length; i++) {
        System.out.println(cgschemas[i]);
        System.out.println("--------------------------------");
      }
      CGSchema cgs1 = cgschemas[0];
      CGSchema cgs2 = cgschemas[1];

      ColumnSchema f11 = cgs1.getSchema().getColumn(0);
      Assert.assertEquals("r1.f1", f11.name);
      Assert.assertEquals(ColumnType.INT, f11.type);
      ColumnSchema f12 = cgs1.getSchema().getColumn(1);
      Assert.assertEquals("r2.r3.f3", f12.name);
      Assert.assertEquals(ColumnType.FLOAT, f12.type);

      ColumnSchema f21 = cgs2.getSchema().getColumn(0);
      Assert.assertEquals("r1.f2", f21.name);
      Assert.assertEquals(ColumnType.INT, f21.type);
      ColumnSchema f22 = cgs2.getSchema().getColumn(1);
      Assert.assertEquals("r2.r3.f4", f22.name);
      Assert.assertEquals(ColumnType.BYTES, f22.type);

      System.out.println("*********** Column Map **********");
      Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
View Full Code Here
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
      Path parent = path.getParent();
      Path tmpPath1 = new Path(parent, "_temporary");
      Path tmpPath2 = new Path(tmpPath1, name);
      checkPath(tmpPath2, true);
     
      cgschema = new CGSchema(schema, sorted, comparator, name, serializer, compressor, owner, group, perm);
      CGSchema sfNew = CGSchema.load(fs, path);
      if (sfNew != null) {
        // sanity check - compare input with on-disk schema.
        if (!sfNew.equals(cgschema)) {
          throw new IOException("Schema passed in is different from the one on disk");
        }
      } else {
        // create the schema file in FS
        cgschema.create(fs, path);
View Full Code Here
242
243
244
245
246
247
248
249
     */

    public static Schema getSchema(Path path, Configuration conf)
        throws IOException, ParseException {
      FileSystem fs = path.getFileSystem(conf);
      CGSchema cgschema = CGSchema.load(fs, path);
      return cgschema.getSchema();
    }
View Full Code Here
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
        sorted = schemaFile.isSorted();
        colGroups = new ColumnGroup.Writer[numCGs];
        cgTuples = new Tuple[numCGs];
        Path tmpWorkPath = new Path(path, "_temporary"); 
        for (int nx = 0; nx < numCGs; nx++) {
          CGSchema cgschema = new CGSchema(schemaFile.getPhysicalSchema(nx), sorted,
              schemaFile.getComparator(), schemaFile.getName(nx), schemaFile.getSerializer(nx), schemaFile.getCompressor(nx),
              schemaFile.getOwner(nx), schemaFile.getGroup(nx), schemaFile.getPerm(nx));
         
          colGroups[nx] =
            new ColumnGroup.Writer(
View Full Code Here
TOP

Related Classes of org.apache.hadoop.zebra.types.CGSchema

  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.FSDataOutputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.io.file.tfile.Utils.Version
  • org.apache.hadoop.zebra.io.BasicTable$Writer
  • org.apache.hadoop.zebra.io.ColumnGroup$Reader
  • org.apache.hadoop.zebra.io.ColumnGroup$Writer
  • org.apache.hadoop.zebra.schema.Schema
  • org.apache.hadoop.zebra.tfile.Utils.Version
  • org.apache.hadoop.zebra.types.Schema
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.