Package org.apache.hadoop.zebra.io

Examples of org.apache.hadoop.zebra.io.TableInserter


    }

    for (int i = 0; i < parts; ++i) {
      Path workPath = new Path(path.getParent(), "_temporary");
      writer = new ColumnGroup.Writer(path, workPath, conf);
      TableInserter inserter = writer.getInserter(String.format("part-%06d",
          permutation[i]), true);
      if ((rows > 0) && !emptyTFileSet.contains(permutation[i])) {
        int actualRows = random.nextInt(rows) + rows / 2;
        for (int j = 0; j < actualRows; ++j, ++total) {
          BytesWritable key;
          if (!sorted) {
            key = makeRandomKey(rows * 10);
          } else {
            key = makeKey(total);
          }
          TypesUtils.resetTuple(tuple);
          for (int k = 0; k < tuple.size(); ++k) {
            try {
              tuple.set(k, makeString("col-" + colNames[k], rows * 10));
            } catch (ExecException e) {
              e.printStackTrace();
            }
          }
          inserter.insert(key, tuple);
        }
      }
      inserter.close();
    }

    if (properClose) {
      writer = new ColumnGroup.Writer(path, conf);
      writer.close();
View Full Code Here


      permutation[targetIndex] = tmp;
    }

    for (int i = 0; i < parts; ++i) {
      writer = new ColumnGroup.Writer(path, conf);
      TableInserter inserter = writer.getInserter(String.format("part-%06d",
          permutation[i]), true);
      if (rows > 0) {
        int actualRows = random.nextInt(rows * 2 / 3) + rows * 2 / 3;
        for (int j = 0; j < actualRows; ++j, ++total) {
          BytesWritable key = keyGen.next();
          TypesUtils.resetTuple(tuple);
          for (int k = 0; k < tuple.size(); ++k) {
            try {
              tuple.set(k, makeString("col-" + colNames[k], rows * 10));
            } catch (ExecException e) {
              e.printStackTrace();
            }
          }
          inserter.insert(key, tuple);
        }
      }
      inserter.close();
    }

    writer = new ColumnGroup.Writer(path, conf);
    writer.close();
    BasicTableStatus status = getStatus(path);
View Full Code Here

    writer.finish();
    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);
    BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
    int part = 0;
    TableInserter inserter = writer1.getInserter("part" + part, true);
    TypesUtils.resetTuple(tuple);

    Tuple tupRecord1;
    try {
      tupRecord1 = TypesUtils.createTuple(schema.getColumnSchema("r1")
          .getSchema());
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }

    Tuple tupRecord2;
    try {
      tupRecord2 = TypesUtils.createTuple(schema.getColumnSchema("r2")
          .getSchema());
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }

    Tuple tupRecord3;
    try {
      tupRecord3 = TypesUtils.createTuple(new Schema("f3:float, f4:map(int)"));
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }
    HashMap<String, Float> map1 = new HashMap<String, Float>();
    // insert data in row 1
    int row = 0;
    // r1:record(f1:int, f2:long)
    tupRecord1.set(0, 1);
    tupRecord1.set(1, 1001L);
    tuple.set(0, tupRecord1);

    // r2:record(r3:record(f3:map(float), map(int)))
    HashMap<String, Integer> map = new HashMap<String, Integer>();
    tupRecord2.set(0, tupRecord3);
    map1.put("x", 1.3F);
    tupRecord3.set(0, map1);
    map.put("a", 1);
    map.put("b", 2);
    map.put("c", 3);
    tupRecord3.set(1, map);
    tuple.set(1, tupRecord2);
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

    // row 2
    map1.clear();
    row++;
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(tupRecord1);
    TypesUtils.resetTuple(tupRecord2);
    TypesUtils.resetTuple(tupRecord3);
    // r1:record(f1:int, f2:map(long))
    tupRecord1.set(0, 2);
    tupRecord1.set(1, 1002L);
    tuple.set(0, tupRecord1);

    // r2:record(r3:record(f3:float, f4))
    tupRecord2.set(0, tupRecord3);
    map.clear();
    map.put("x", 11);
    map.put("y", 12);
    map.put("c", 13);

    map1.put("y", 2.3F);
    tupRecord3.set(0, map1);
    tupRecord3.set(1, map);
    tuple.set(1, tupRecord2);

    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

    // finish building table, closing out the inserter, writer, writer1
    inserter.close();
    writer1.finish();
    writer.close();
  }
View Full Code Here

    writer.finish();
    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);
    BasicTable.Writer writer1 = new BasicTable.Writer(pathTable, conf);
    int part = 0;
    TableInserter inserter = writer1.getInserter("part" + part, true);
    TypesUtils.resetTuple(tuple);

    Tuple record1;
    try {
      record1 = TypesUtils.createTuple(new Schema("f1:int, f2:string"));
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }

    Tuple record2;
    try {
      record2 = TypesUtils.createTuple(new Schema("f1:int, f2:string"));
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }

    Tuple record3;
    try {
      record3 = TypesUtils.createTuple(new Schema("f1:int, f2:string"));
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }

    // add data to row 1
    // m1:map(string)
    Map<String, String> m1 = new HashMap<String, String>();
    m1.put("a", "A");
    m1.put("b", "B");
    m1.put("c", "C");
    tuple.set(0, m1);

    // m2:map(map(int))
    HashMap<String, Map> m2 = new HashMap<String, Map>();
    Map<String, Integer> m31 = new HashMap<String, Integer>();
    m31.put("m311", 311);
    m31.put("m321", 321);
    m31.put("m331", 331);
    Map<String, Integer> m32 = new HashMap<String, Integer>();
    m32.put("m411", 411);
    m32.put("m421", 421);
    m32.put("m431", 431);
    m2.put("x", m31);
    m2.put("y", m32);
    tuple.set(1, m2);

    // m4:map(map(record(f1:int,f2:string)))
    record1.set(0, 11);
    record1.set(1, "record row 1.1");
    Map<String, Tuple> m51 = new HashMap<String, Tuple>();
    Map<String, Tuple> m52 = new HashMap<String, Tuple>();
    Map<String, Tuple> m53 = new HashMap<String, Tuple>();
    m51.put("ma4", (Tuple) record1);
    m52.put("ma41", (Tuple) record1);
    m53.put("ma43", (Tuple) record1);

    record2.set(0, 12);
    record2.set(1, "record row 1.2");
    m51.put("mb4", (Tuple) record2);
    m52.put("mb42", (Tuple) record2);
    m53.put("ma43", (Tuple) record2);
    System.out.println("record1-1: " + record1.toString());

    record3.set(0, 13);
    record3.set(1, "record row 1.3");
    System.out.println("record1-3: " + record1.toString());

    m51.put("mc4", (Tuple) record3);
    m52.put("mc42", (Tuple) record3);
    m53.put("ma43", (Tuple) record3);

    Map<String, Map> m4 = new HashMap<String, Map>();
    m4.put("a4", m51);
    m4.put("b4", m52);
    m4.put("c4", m53);
    m4.put("d4", m53);
    m4.put("ma43", m53);

    tuple.set(2, m4);

    int row = 0;
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

    // row 2
    row++;
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(record1);
    TypesUtils.resetTuple(record2);
    TypesUtils.resetTuple(record3);
    m1.clear();
    m2.clear();
    m31.clear();
    m32.clear();
    m4.clear();
    m51.clear();
    m52.clear();
    m53.clear();
    // m1:map(string)
    m1.put("a", "A2");
    m1.put("b2", "B2");
    m1.put("c2", "C2");
    tuple.set(0, m1);

    // m2:map(map(int))
    m31.put("m321", 321);
    m31.put("m322", 322);
    m31.put("m323", 323);
    m2.put("z", m31);
    tuple.set(1, m2);

    // m4:map(map(record(f1:int,f2:string)))
    record1.set(0, 21);
    record1.set(1, "record row 2.1");
    m51.put("ma4", (Tuple) record1);
    m52.put("ma41", (Tuple) record1);
    m53.put("ma43", (Tuple) record1);

    record2.set(0, 22);
    record2.set(1, "record row 2.2");
    m51.put("mb4", (Tuple) record2);
    m52.put("mb42", (Tuple) record2);
    m53.put("ma43", (Tuple) record2);

    record3.set(0, 33);
    record3.set(1, "record row 3.3");
    m51.put("mc4", (Tuple) record3);
    m52.put("mc42", (Tuple) record3);
    m53.put("ma43", (Tuple) record3);

    m4.put("a4", m51);
    m4.put("b4", m52);

    m4.put("ma43", m53);

    tuple.set(2, m4);

    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

    // finish building table, closing out the inserter, writer, writer1
    inserter.close();
    writer1.finish();
    writer.close();
  }
View Full Code Here

    writer.finish();
    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);
    BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
    int part = 0;
    TableInserter inserter = writer1.getInserter("part" + part, true);
    TypesUtils.resetTuple(tuple);

    Tuple tupRecord1;
    try {
      tupRecord1 = TypesUtils.createTuple(schema.getColumnSchema("r1")
          .getSchema());
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }

    Tuple tupRecord2;
    try {
      tupRecord2 = TypesUtils.createTuple(schema.getColumnSchema("r2")
          .getSchema());
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }

    Tuple tupRecord3;
    try {
      tupRecord3 = TypesUtils.createTuple(new Schema("f3:float, f4"));
    } catch (ParseException e) {
      e.printStackTrace();
      throw new IOException(e);
    }

    // insert data in row 1
    int row = 0;
    // r1:record(f1:int, f2:long
    tupRecord1.set(0, 1);
    tupRecord1.set(1, 1001L);
    tuple.set(0, tupRecord1);

    // r2:record(r3:record(f3:float, f4))
    tupRecord2.set(0, "f5 row1 byte array");

    tupRecord3.set(0, 1.3);
    tupRecord3.set(1, new DataByteArray("r3 row1 byte array"));
    tupRecord2.set(1, tupRecord3);
    tuple.set(1, tupRecord2);
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

    // row 2
    row++;
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(tupRecord1);
    TypesUtils.resetTuple(tupRecord2);
    TypesUtils.resetTuple(tupRecord3);
    // r1:record(f1:int, f2:long
    tupRecord1.set(0, 2);
    tupRecord1.set(1, 1002L);
    tuple.set(0, tupRecord1);

    // r2:record(r3:record(f3:float, f4))
    tupRecord2.set(0, "f5 row2 byte array");
    tupRecord3.set(0, 2.3);
    tupRecord3.set(1, new DataByteArray("r3 row2 byte array"));
    tupRecord2.set(1, tupRecord3);
    tuple.set(1, tupRecord2);

    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

    // finish building table, closing out the inserter, writer, writer1
    inserter.close();
    writer1.finish();
    writer.close();
  }
View Full Code Here

    writer.finish();
    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);
    BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
    int part = 0;
    TableInserter inserter = writer1.getInserter("part" + part, true);
    TypesUtils.resetTuple(tuple);

    DataBag bag2 = TypesUtils.createBag();
    Schema schColl2 = schema.getColumn(7).getSchema();
    Tuple tupColl2_1 = TypesUtils.createTuple(schColl2);
    Tuple tupColl2_2 = TypesUtils.createTuple(schColl2);
    // add data to row 1
    tuple.set(0, new DataByteArray("column1 row 1 ")); // byte
    tuple.set(1,new DataByteArray("column2 row 1"));
    tuple.set(2, new DataByteArray("column3 row 1"));
    tuple.set(3, new DataByteArray("column4 row 1"));
   
    // column5
    Map<String, String> column5 = new HashMap<String, String>();
    column5.put("key51", "key511");
    column5.put("key52", "key521");
    column5.put("key53", "key531");
    column5.put("key54", "key541");
    column5.put("key55", "key551");
    column5.put("key56", "key561");
    column5.put("key57", "key571");
  
    tuple.set(4, column5);

    //column5:map(bytes),column6:map(bytes),column7:map(bytes),column8:collection(f1:map(bytes)
    //column7:map(String, column6:map(String)
    HashMap<String, String> column7 = new HashMap<String, String>();
    HashMap<String, String> column6 = new HashMap<String, String>();
    column6.put("column61", "column61");
    column7.put("key71", "key711");
    column7.put("key72", "key721");
    column7.put("key73", "key731");
    column7.put("key74", "key741");
    column7.put("key75", "key751");
    tuple.set(6, column7);
    tuple.set(5, column6);
   
    //column8:collection(f1:map(bytes))
    HashMap<String, String> mapInCollection = new HashMap<String, String>();
    mapInCollection.put("mc", "mc1");
    tupColl2_1.set(0, mapInCollection);
    bag2.add(tupColl2_1);

    tupColl2_2.set(0, mapInCollection);
    bag2.add(tupColl2_2);
    tuple.set(7, bag2);

    int row = 0;
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

    // row 2
    row++;
    TypesUtils.resetTuple(tuple);
    column5.clear();
    column7.clear();
    column6.clear();
    mapInCollection.clear();
    bag2.clear();
    TypesUtils.resetTuple(tupColl2_1);
    TypesUtils.resetTuple(tupColl2_2);
   
    tuple.set(0, new DataByteArray("column1 row 2 ")); // byte
    tuple.set(1,new DataByteArray("column2 row 2"));
    tuple.set(2, new DataByteArray("column3 row 2"));
    tuple.set(3, new DataByteArray("column4 row 2"));
   
    // column5
    column5.put("key51", "key512");
    column5.put("key52", "key522");
    column5.put("key53", "key532");
    column5.put("key54", "key542");
    column5.put("key55", "key552");
    column5.put("key56", "key562");
    column5.put("key57", "key572");
    tuple.set(4, column5);

    // column6
 
    column6.put("column6", "column62");
    column7.put("key71", "key712");
    column7.put("key72", "key722");
    column7.put("key73", "key732");
    column7.put("key74", "key742");
    column7.put("key75", "key752");
    tuple.set(6, column7);
    tuple.set(5, column6);
   
   
    //column8
    //column8:collection(f1:map(bytes))
    mapInCollection.put("mc", "mc2");
    tupColl2_1.set(0, mapInCollection);
    bag2.add(tupColl2_1);

    tupColl2_2.set(0, mapInCollection);
    bag2.add(tupColl2_2);
    tuple.set(7, bag2);

   
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);

    // finish building table, closing out the inserter, writer, writer1
    inserter.close();
    writer1.finish();
    writer.close();
  }
View Full Code Here

    BasicTable.Writer writer = new BasicTable.Writer(unsortedPath,
        TABLE_SCHEMA, TABLE_STORAGE, conf);

    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);
    TableInserter inserter = writer.getInserter("ins", false);

    Map<String, String> m1 = new HashMap<String, String>();

    Tuple tupRecord1; // record
    tupRecord1 = TypesUtils.createTuple(schema.getColumnSchema("r1")
        .getSchema()); // r1 schema

    DataBag bag1 = TypesUtils.createBag();
    Schema schColl = schema.getColumnSchema("c1").getSchema(); // c1 schema
    Tuple tupColl1 = TypesUtils.createTuple(schColl);
    Tuple tupColl2 = TypesUtils.createTuple(schColl);

    int randRange = new Long(numbRows / 10).intValue(); // random range to allow
    // for duplicate values
    for (int i = 0; i < numbRows; ++i) {
      int random = generator.nextInt(randRange);

      TypesUtils.resetTuple(tuple); // reset row tuple
      m1.clear(); // reset map
      TypesUtils.resetTuple(tupRecord1); // reset record
      TypesUtils.resetTuple(tupColl1); // reset collection
      TypesUtils.resetTuple(tupColl2);
      bag1.clear();

      tuple.set(0, i); // count
      tuple.set(1, seed); // seed

      tuple.set(2, i); // int1
      tuple.set(3, random); // int2
      tuple.set(4, "string " + i); // str1
      tuple.set(5, "string random " + random); // str2
      tuple.set(6, new DataByteArray("byte " + i)); // byte1
      tuple.set(7, new DataByteArray("byte random " + random)); // byte2

      tuple.set(8, new Float(i * -1)); // float1 negative
      tuple.set(9, new Long(numbRows - i)); // long1 reverse
      tuple.set(10, new Double(i * 100)); // double1

      // insert map1
      m1.put("a", "m1");
      m1.put("b", "m1 " + i);
      tuple.set(11, m1);

      // insert record1
      tupRecord1.set(0, "r1 " + seed);
      tupRecord1.set(1, "r1 " + i);
      tuple.set(12, tupRecord1);

      // insert collection1
      // tupColl1.set(0, "c1 a " + seed);
      // tupColl1.set(1, "c1 a " + i);
      // bag1.add(tupColl1); // first collection item
      bag1.add(tupRecord1); // first collection item
      bag1.add(tupRecord1); // second collection item

      // tupColl2.set(0, "c1 b " + seed);
      // tupColl2.set(1, "c1 b " + i);
      // bag1.add(tupColl2); // second collection item

      tuple.set(13, bag1);

      inserter.insert(new BytesWritable(("key" + i).getBytes()), tuple);
    }
    inserter.close();
    writer.close();

    if (debug == true) {
      // Load tables
      String query1 = "table1 = LOAD '" + unsortedPath.toString()
View Full Code Here

  public void testInsertNullValues() throws IOException, ParseException {
    fs.delete(path, true);
    System.out.println("testInsertNullValues");
    writer = new ColumnGroup.Writer(path, "abc, def", false, path.getName(), "pig", "gz",
        null, null, (short) -1, true, conf);
    TableInserter ins = writer.getInserter("part1", true);
    // Tuple row = TypesUtils.createTuple(writer.getSchema());
    // ins.insert(new BytesWritable("key".getBytes()), row);
    ins.close();
    close();
  }
View Full Code Here

  public void testFailureInvalidSchema() throws IOException, ParseException {
    fs.delete(path, true);
    System.out.println("testFailureInvalidSchema");
    writer = new ColumnGroup.Writer(path, "abc, def", false, path.getName(), "pig", "gz",
        null, null, (short) -1, true, conf);
    TableInserter ins = writer.getInserter("part1", true);
    Tuple row = TypesUtils.createTuple(Schema.parse("xyz, ijk, def"));
    try {
      ins.insert(new BytesWritable("key".getBytes()), row);
      Assert.fail("Failed to catch diff schemas.");
    } catch (IOException e) {
      // noop, expecting exceptions
    } finally {
      ins.close();
      close();
    }
  }
View Full Code Here

    System.out.println("testFailureGetInserterAfterWriterClosed");
    writer = new ColumnGroup.Writer(path, "abc, def", false, path.getName(), "pig", "gz",
        null, null, (short) -1, true, conf);
    try {
      writer.close();
      TableInserter ins = writer.getInserter("part1", true);
      Assert.fail("Failed to catch getInsertion after writer closure.");
    } catch (IOException e) {
      // noop, expecting exceptions
    } finally {
      close();
View Full Code Here

TOP

Related Classes of org.apache.hadoop.zebra.io.TableInserter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.