Examples of org.apache.hadoop.io.BytesWritable

org.apache.hadoop.io.BytesWritable
A byte sequence that is usable as a key or value. It is resizable and distinguishes between the size of the seqeunce and the current capacity. The hash function is the front of the md5 of the buffer. The sort order is the same as memcmp. @author Doug Cutting

        // ensure the TFile index is loaded throughout the life of scanner.
        reader.checkTFileDataIndex();
        beginLocation = begin;
        endLocation = end;


        valTransferBuffer = new BytesWritable();
        // TODO: remember the longest key in a TFile, and use it to replace
        // MAX_KEY_SIZE.
        keyBuffer = new byte[MAX_KEY_SIZE];
        keyDataInputStream = new DataInputBuffer();
        valueBufferInputStream = new ChunkDecoder();

View Full Code Here

            tuple.set(k, (9-b) + "_" + i + "" + k);
          } catch (ExecException e) {
            e.printStackTrace();
          }
        }
        inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
      }
    }
    for (int i = 0; i < numsInserters; i++) {
      inserters[i].close();
    }

View Full Code Here

      TypesUtils.resetTuple(tuple);
      for (int k = 0; k < tableData[i].length; ++k) {
        tuple.set(k, tableData[i][k]);
        System.out.println("DEBUG: setting tuple k=" + k + "value= " + tableData[i][k]);
      }
      inserter.insert(new BytesWritable(("key" + i).getBytes()), tuple);
    }
    inserter.close();
    writer.close();
  }

View Full Code Here

            tuple.set(k, (9-b) + "_" + i + "" + k);
          } catch (ExecException e) {
            e.printStackTrace();
          }
        }
        inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
      }
    }
    for (int i = 0; i < numsInserters; i++) {
      inserters[i].close();
    }

View Full Code Here

  
        } catch (ExecException e) {
          e.printStackTrace();
        }
  
       inserters[i].insert(new BytesWritable(("key_" + b).getBytes()), tuple);
       
      }
    }
    for (int i = 0; i < numsInserters; i++) {
      inserters[i].close();
    }
    writer.close();
    
  
    //check table is setup correctly
    String projection = new String("a,b,c,d,e,f,r1,m1");
    
    BasicTable.Reader reader = new BasicTable.Reader(pathTable1, conf);
    reader.setProjection(projection);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());
  
    scanner.getValue(RowValue);
    System.out.println("rowvalue size:"+RowValue.size());
    System.out.println("read a : " + RowValue.get(0).toString());

View Full Code Here

    
    BasicTable.Reader reader = new BasicTable.Reader(pathTable2, conf);
    reader.setProjection(projection);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());


    /*
    scanner.getValue(RowValue);
    System.out.println("rowvalue size:"+RowValue.size());

View Full Code Here

    tupRecord1.set(0, "1");
    tupRecord1.set(1, "hello1");
    tuple.set(2, tupRecord1);
    tuple.set(3, "world1");


    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    // second row
    row++;
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(tupRecord1);
    TypesUtils.resetTuple(tupColl1);
    TypesUtils.resetTuple(tupColl2);
    m1.clear();
    bag1.clear();


    row++;
    tupColl1.set(0, "2.2");
    tupColl1.set(1, "2.22");
    bag1.add(tupColl1);
    tupColl2.set(0, "2.222");
    tupColl2.set(1, "2.2222");
    bag1.add(tupColl2);
    tuple.set(0, bag1);


    m1.put("k2", "k22");
    m1.put("k3", "k32");
    m1.put("k1", "k12");
    m1.put("k4", "k42");
    tuple.set(1, m1);


    tupRecord1.set(0, "2");
    tupRecord1.set(1, "hello2");
    tuple.set(2, tupRecord1);
    tuple.set(3, "world2");


    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);
    inserter.close();
    writer1.finish();
    writer.close();


    /*
     * create 2nd basic table;
     */
    pathTable2 = new Path(pathWorking, "2");
    System.out.println("pathTable2 =" + pathTable2);


    BasicTable.Writer writer2 = new BasicTable.Writer(pathTable2, STR_SCHEMA2,
        STR_STORAGE2, conf);
    Schema schema2 = writer.getSchema();


    Tuple tuple2 = TypesUtils.createTuple(schema2);


    BasicTable.Writer writer22 = new BasicTable.Writer(pathTable2, conf);
    part = 0;
    TableInserter inserter2 = writer22.getInserter("part" + part, true);


    TypesUtils.resetTuple(tuple2);
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(tupRecord1);
    TypesUtils.resetTuple(tupColl1);
    TypesUtils.resetTuple(tupColl2);
    m1.clear();
    bag1.clear();


    row = 0;
    tupColl1.set(0, "3.3");
    tupColl1.set(1, "3.33");
    bag1.add(tupColl1);
    tupColl2.set(0, "3.333");
    tupColl2.set(1, "3.3333");
    bag1.add(tupColl2);
    tuple2.set(0, bag1);


    m1.put("k1", "k13");
    m1.put("b", "b3");
    m1.put("c", "c3");
    tuple2.set(1, m1);


    tupRecord1.set(0, "3");
    tupRecord1.set(1, "hello3");
    tuple2.set(2, tupRecord1);
    tuple2.set(3, "world13");


    inserter2.insert(new BytesWritable(String
        .format("k%d%d", part + 1, row + 1).getBytes()), tuple2);


    // second row
    row++;
    TypesUtils.resetTuple(tuple2);
    TypesUtils.resetTuple(tupRecord1);
    TypesUtils.resetTuple(tupColl1);
    TypesUtils.resetTuple(tupColl2);
    bag1.clear();
    m1.clear();


    row++;
    tupColl1.set(0, "4.4");
    tupColl1.set(1, "4.44");
    bag1.add(tupColl1);
    tupColl2.set(0, "4.444");
    tupColl2.set(1, "4.4444");
    bag1.add(tupColl2);
    tuple2.set(0, bag1);


    m1.put("k2", "k24");
    m1.put("k3", "k34");
    m1.put("k1", "k14");
    m1.put("k4", "k44");
    tuple2.set(1, m1);


    tupRecord1.set(0, "4");
    tupRecord1.set(1, "hello4");
    tuple2.set(2, tupRecord1);
    tuple2.set(3, "world4");


    inserter2.insert(new BytesWritable(String
        .format("k%d%d", part + 1, row + 1).getBytes()), tuple2);
    inserter2.close();
    writer2.finish();
    writer22.close();

View Full Code Here

    for (int i = 7; i <= 23; i++) {
      tuple.set(i + 4, "s" + "i" + ", line1");
    }


    int row = 0;
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    // row 2
    row++;
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(tupRecord1);
    TypesUtils.resetTuple(tupRecord2);
    TypesUtils.resetTuple(tupRecord3);
    m1.clear();
    m2.clear();
    m3.clear();
    m4.clear();
    tuple.set(0, false);
    tuple.set(1, 2); // int
    tuple.set(2, 1002L); // long
    tuple.set(3, 3.1); // float
    tuple.set(4, "hello world 2"); // string
    tuple.set(5, new DataByteArray("hello byte 2")); // byte


    // r1:record(f1:int, f2:long
    tupRecord1.set(0, 2);
    tupRecord1.set(1, 1002L);
    tuple.set(6, tupRecord1);


    // r2:record(r3:record(f3:float, f4))
    tupRecord2.set(0, tupRecord3);
    tupRecord3.set(0, 2.3);
    tupRecord3.set(1, new DataByteArray("r3 row2  byte array"));
    tuple.set(7, tupRecord2);


    // m1:map(string)
    m1.put("a2", "A2");
    m1.put("b2", "B2");
    m1.put("c2", "C2");
    tuple.set(8, m1);


    // m2:map(map(int))
    m3.put("m321", 321);
    m3.put("m322", 322);
    m3.put("m323", 323);
    m2.put("z", m3);
    tuple.set(9, m2);


    // c:collection(f13:double, f14:float, f15:bytes)
    bagColl.clear();
    TypesUtils.resetTuple(tupColl1);
    TypesUtils.resetTuple(tupColl2);
    tupColl1.set(0, 7654.321);
    tupColl1.set(1, 0.0001);
    abs1[0] = 31;
    abs1[1] = 32;
    abs1[2] = 33;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 0.123456789);
    tupColl2.set(1, 0.3333);
    abs2[0] = 41;
    abs2[1] = 42;
    abs2[2] = 43;
    abs2[3] = 44;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    tuple.set(10, bagColl);
    // set s7 to s23
    for (int i = 7; i <= 23; i++) {
      tuple.set(i + 4, "s" + "i" + ", line2");
    }
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    inserter.close();
    writer1.finish();

View Full Code Here

      try {
        indexReader = TableInputFormat.getTableRecordReader(jobConf, null);
      } catch (ParseException e) {
        throw new IOException("Exception from TableInputFormat.getTableRecordReader: "+e.getMessage());
      }
      indexKey = new BytesWritable();
    }

View Full Code Here

    byte[] types = new byte[sortColNames.length];
    for(int i =0 ; i < sortColNames.length; ++i){
      types[i] = schema.getColumn(sortColNames[i]).getType().pigDataType();
    }
    KeyGenerator builder = makeKeyBuilder(types);
    BytesWritable key = builder.generateKey(t);
//      BytesWritable key = new BytesWritable(((String) t.get(0)).getBytes());
    indexReader.seekTo(key);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.BytesWritable

com.backtype.cascading.tap.PailTap$PailScheme

com.backtype.hadoop.pail.SequenceFileFormat$SequenceFilePailRecordReader

com.cloudera.sqoop.testutil.ManagerCompatTestCase

com.facebook.hive.orc.OrcTestUtils

com.facebook.hive.orc.TestOrcFile

com.facebook.hive.orc.WriterImpl$BinaryTreeWriter

com.facebook.presto.hive.util.SerDeUtils

com.pinterest.secor.tools.LogFileVerifier

edu.yale.cs.hadoopdb.connector.DBInputSplit

hipi.imagebundle.SeqImageBundle

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.