Package org.apache.hadoop.zebra.types

Examples of org.apache.hadoop.zebra.types.Schema

75
76
77
78
79
80
81
82
83
84
85
    pathTable = new Path(pathWorking, "TestTableLoader");
    System.out.println("pathTable =" + pathTable);

    BasicTable.Writer writer = new BasicTable.Writer(pathTable,
        "a:string,b,c:string,d,e,f,g", "[a,b,c];[d,e,f,g]", false, conf);
    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);

    final int numsBatch = 10;
    final int numsInserters = 2;
    TableInserter[] inserters = new TableInserter[numsInserters];
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    pathTable = new Path(pathWorking, "TestTableStorer");
    System.out.println("pathTable =" + pathTable);
    BasicTable.Writer writer = new BasicTable.Writer(pathTable,
        "SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
        "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);

    final int numsBatch = 10;
    final int numsInserters = 2;
    TableInserter[] inserters = new TableInserter[numsInserters];
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
     * } for (int i = 0; i < numsInserters; i++) { inserters[i].close(); }
     */

    writer.finish();

    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);

    BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
    int part = 0;
    TableInserter inserter = writer1.getInserter("part" + part, true);
    TypesUtils.resetTuple(tuple);
    DataBag bagColl = TypesUtils.createBag();
    Schema schColl = schema.getColumn(0).getSchema();
    Tuple tupColl1 = TypesUtils.createTuple(schColl);
    Tuple tupColl2 = TypesUtils.createTuple(schColl);
    byte[] abs1 = new byte[3];
    byte[] abs2 = new byte[4];
    tupColl1.set(0, 3.1415926);
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    ColumnSchema f1 = schema.getColumn(0);
    Assert.assertEquals("r", f1.name);
    Assert.assertEquals(ColumnType.RECORD, f1.type);

    // test 2nd level schema;
    Schema f1Schema = f1.schema;
    ColumnSchema f11 = f1Schema.getColumn(0);
    Assert.assertEquals("r", f11.name);
    Assert.assertEquals(ColumnType.RECORD, f11.type);
    ColumnSchema f12 = f1Schema.getColumn(1);
    Assert.assertEquals("f2", f12.name);
    Assert.assertEquals(ColumnType.MAP, f12.type);

    // test 3rd level schema;
    Schema f11Schema = f11.schema;
    ColumnSchema f111 = f11Schema.getColumn(0);
    Assert.assertEquals("f1", f111.name);
    Assert.assertEquals(ColumnType.INT, f111.type);
    ColumnSchema f112 = f11Schema.getColumn(1);
    Assert.assertEquals("f2", f112.name);
    Assert.assertEquals(ColumnType.INT, f112.type);

    Schema f12Schema = f12.schema;
    ColumnSchema f121 = f12Schema.getColumn(0);
    // Assert.assertEquals("", f121.name);
    Assert.assertEquals(ColumnType.BYTES, f121.type);
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    ColumnSchema f2 = schema.getColumn(1);
    Assert.assertEquals("m1", f2.name);
    Assert.assertEquals(ColumnType.MAP, f2.type);

    // test 2nd level schema;
    Schema f1Schema = f1.schema;
    ColumnSchema f11 = f1Schema.getColumn(0);
    Assert.assertEquals("r", f11.name);
    Assert.assertEquals(ColumnType.RECORD, f11.type);

    Schema f2Schema = f2.schema;
    ColumnSchema f21 = f2Schema.getColumn(0);
    // Assert.assertEquals("", f21.name);
    Assert.assertEquals(ColumnType.INT, f21.type);

    // test 3rd level schema;
    Schema f11Schema = f11.schema;
    ColumnSchema f111 = f11Schema.getColumn(0);
    Assert.assertEquals("r", f111.name);
    Assert.assertEquals(ColumnType.RECORD, f111.type);
    ColumnSchema f112 = f11Schema.getColumn(1);
    Assert.assertEquals("f2", f112.name);
    Assert.assertEquals(ColumnType.MAP, f112.type);

    // test 4th level schema;
    Schema f111Schema = f111.schema;
    ColumnSchema f1111 = f111Schema.getColumn(0);
    Assert.assertEquals("f1", f1111.name);
    Assert.assertEquals(ColumnType.INT, f1111.type);
    ColumnSchema f1112 = f111Schema.getColumn(1);
    Assert.assertEquals("f2", f1112.name);
    Assert.assertEquals(ColumnType.INT, f1112.type);

    Schema f112Schema = f112.schema;
    ColumnSchema f1121 = f112Schema.getColumn(0);
    // Assert.assertEquals("", f1121.name);
    Assert.assertEquals(ColumnType.BYTES, f1121.type);
  }
View Full Code Here
952
953
954
955
956
957
958
959
960
961
962
963
  void printFreqWords() throws IOException, ParseException {
    LOG.info("Printing the most frequent words");
    BasicTable.Reader reader = new BasicTable.Reader(freqWordTablePath, conf);
    TableScanner scanner = reader.getScanner(null, true);
    BytesWritable key = new BytesWritable();
    Schema schema = Projection.toSchema(scanner.getProjection());
    int idxCount = schema.getColumnIndex("count");
    Tuple value = TypesUtils.createTuple(schema);
    for (; !scanner.atEnd(); scanner.advance()) {
      scanner.getKey(key);
      scanner.getValue(value);
      try {
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
      @Override
      public void configure(JobConf job) {
        LOG.info("ForwardIndexGen.MapClass.configure");
        outKey = new BytesWritable();
        try {
          Schema outSchema = BasicTableOutputFormat.getSchema(job);
          outRow = TypesUtils.createTuple(outSchema);
          idxFileName = outSchema.getColumnIndex("fileName");
          idxWordPos = outSchema.getColumnIndex("wordPos");
          idxLineNo = outSchema.getColumnIndex("lineNo");
        } catch (IOException e) {
          throw new RuntimeException("Schema parsing failed : "
              + e.getMessage());
        } catch (ParseException e) {
          throw new RuntimeException("Schema parsing failed : "
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
public class TestSchemaMap {
  @Test
  public void testSchemaValid1() throws ParseException {
    String strSch = "f1:int, m1:map(int)";
    TableSchemaParser parser;
    Schema schema;

    parser = new TableSchemaParser(new StringReader(strSch));
    schema = parser.RecordSchema(null);
    System.out.println(schema);

    // test 1st level schema;
    ColumnSchema f1 = schema.getColumn(0);
    Assert.assertEquals("f1", f1.name);
    Assert.assertEquals(ColumnType.INT, f1.type);

    ColumnSchema f2 = schema.getColumn(1);
    Assert.assertEquals("m1", f2.name);
    Assert.assertEquals(ColumnType.MAP, f2.type);
  }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
  @Test
  public void testSchemaValid2() throws ParseException {
    String strSch = "f1:int, m1:map(map(float))";
    TableSchemaParser parser;
    Schema schema;

    parser = new TableSchemaParser(new StringReader(strSch));
    schema = parser.RecordSchema(null);
    System.out.println(schema);

    // test 1st level schema;
    ColumnSchema f1 = schema.getColumn(0);
    Assert.assertEquals("f1", f1.name);
    Assert.assertEquals(ColumnType.INT, f1.type);

    ColumnSchema f2 = schema.getColumn(1);
    Assert.assertEquals("m1", f2.name);
    Assert.assertEquals(ColumnType.MAP, f2.type);

    // test 2nd level schema;
    Schema f2Schema = f2.schema;
    ColumnSchema f21 = f2Schema.getColumn(0);
    // Assert.assertEquals("m1", f2.name);
    Assert.assertEquals(ColumnType.MAP, f21.type);

    // test 3rd level schema;
    Schema f21Schema = f21.schema;
    ColumnSchema f211 = f21Schema.getColumn(0);
    // Assert.assertEquals("m1", f2.name);
    Assert.assertEquals(ColumnType.FLOAT, f211.type);
  }
View Full Code Here
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
      @Override
      public void configure(JobConf job) {
        LOG.info("InvertedIndexGen.ReduceClass.configure");
        try {
          Schema outSchema = BasicTableOutputFormat.getSchema(job);
          outRow = TypesUtils.createTuple(outSchema);
          idxCount = outSchema.getColumnIndex("count");
          idxIndex = outSchema.getColumnIndex("index");
          wordPosSchema = new Schema("wordPos");
          idxWordPos = wordPosSchema.getColumnIndex("wordPos");
        } catch (IOException e) {
          throw new RuntimeException("Schema parsing failed :" + e.getMessage());
        } catch (ParseException e) {
          throw new RuntimeException("Schema parsing failed :" + e.getMessage());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.zebra.types.Schema

  • org.apache.hadoop.zebra.io.BasicTable$Reader
  • org.apache.hadoop.zebra.io.BasicTable$SchemaFile
  • org.apache.hadoop.zebra.io.ColumnGroup$Writer
  • org.apache.hadoop.zebra.io.TestBasicTable
  • org.apache.hadoop.zebra.io.TestBasicTableMapSplits
  • org.apache.hadoop.zebra.io.TestBasicTableProjections
  • org.apache.hadoop.zebra.io.TestBasicTableSplits
  • org.apache.hadoop.zebra.io.TestCollection
  • org.apache.hadoop.zebra.io.TestColumnGroup
  • org.apache.hadoop.zebra.io.TestColumnGroupProjections
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.