Package eu.stratosphere.types

Examples of eu.stratosphere.types.Record$InternalDeSerializer


   
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
      ValueMode.FIX_LENGTH);
   
    // write the records
    Record record = new Record();
    do {
      generator.next(record);
    }
    while (sorter.write(record));
   
    QuickSort qs = new QuickSort();
    qs.sort(sorter);
   
    MutableObjectIterator<Record> iter = sorter.getIterator();
    Record readTarget = new Record();
   
    Value current = new Value();
    Value last = new Value();
   
    iter.next(readTarget);
    readTarget.getFieldInto(1, last);
   
    while ((readTarget = iter.next(readTarget)) != null) {
      readTarget.getFieldInto(1, current);
     
      final int cmp = last.compareTo(current);
      if (cmp > 0) {
        Assert.fail("Next value is not larger or equal to previous value.");
      }
View Full Code Here


    // check order
    final MutableObjectIterator<Record> iterator = sorter.getIterator();
    int pairsEmitted = 1;
    int windowTransitions = 0;
   
    Record rec1 = new Record();
    Record rec2 = new Record();
   
    LOG.debug("Checking results...");
    Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
    while ((rec2 = iterator.next(rec2)) != null)
    {
      final TestData.Key k1 = rec1.getField(0, TestData.Key.class);
      final TestData.Key k2 = rec2.getField(0, TestData.Key.class);
      pairsEmitted++;
     
      // if the next key is smaller again, we have a new window
      if (k1.compareTo(k2) > 0) {
        windowTransitions++;
      }
     
      Record tmp = rec1;
      rec1 = rec2;
      k1.setKey(k2.getKey());
     
      rec2 = tmp;
    }
View Full Code Here

    int numRecs = 50000;
    int[] hit = new int[numChans];

    for (int i = 0; i < numRecs; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
     
      delegate.setInstance(rec);
     
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }

    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] > 0);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecs);

    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, stringComp);

    numChans = 100;
    numRecs = 10000;
   
    hit = new int[numChans];

    for (int i = 0; i < numRecs; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
      delegate.setInstance(rec);
       
      int[] chans = oe2.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
View Full Code Here

   
    int[] hit = new int[numChannels];

    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      delegate.setInstance(rec);
     
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }

    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] == (numRecords/numChannels) || hit[i] == (numRecords/numChannels)-1);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecords);

    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, stringComp);

    numChannels = 100;
    numRecords = 10000;
   
    hit = new int[numChannels];

    for (int i = 0; i < numRecords; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
      delegate.setInstance(rec);
       
      int[] chans = oe2.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
View Full Code Here

   
    int[] hit = new int[numChannels];

    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      delegate.setInstance(rec);
     
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }

    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i]+"", hit[i] == numRecords);
    }
   
    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, stringComp);

    numChannels = 100;
    numRecords = 5000;
   
    hit = new int[numChannels];

    for (int i = 0; i < numRecords; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
      delegate.setInstance(rec);
       
      int[] chans = oe2.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
View Full Code Here

    int numRecords = 5000;
   
    int[] hit = new int[numChannels];

    for (int i = 0; i < numRecords; i++) {
      Record rec = new Record(4);
      rec.setField(0, new IntValue(i));
      rec.setField(1, new StringValue("AB"+i+"CD"+i));
      rec.setField(3, new DoubleValue(i*3.141d));
      delegate.setInstance(rec);
     
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {1}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
   
    Record rec = new Record(0);
    rec.setField(0, new IntValue(1));
    delegate.setInstance(rec);
   
    try {
      oe1.selectChannels(delegate, 100);
    } catch (KeyFieldOutOfBoundsException re) {
View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
   
    Record rec = new Record(2);
    rec.setField(1, new IntValue(1));
    delegate.setInstance(rec);

    try {
      oe1.selectChannels(delegate, 100);
    } catch (NullKeyFieldException re) {
View Full Code Here

    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
   
    PipedInputStream pipedInput = new PipedInputStream(1024*1024);
    DataInputStream in = new DataInputStream(pipedInput);
    DataOutputStream out;
    Record rec = null;
   
    try {
      out = new DataOutputStream(new PipedOutputStream(pipedInput));
     
      rec = new Record(1);
      rec.setField(0, new IntValue());
     
      rec.write(out);
      rec = new Record();
      rec.read(in);
   
    } catch (IOException e) {
      fail("Test erroneous");
    }

View Full Code Here

     
      if (insertInvalidData) {
        bw.write("####_I_AM_INVALID_########\n");
      }
     
      Record rec = new Record();
      while ((rec = inIt.next(rec)) != null) {
        IntValue key = rec.getField(0, IntValue.class);
        IntValue value = rec.getField(1, IntValue.class);
       
        bw.write(key.getValue() + "_" + value.getValue() + "\n");
      }
      if (insertInvalidData) {
        bw.write("####_I_AM_INVALID_########\n");
View Full Code Here

TOP

Related Classes of eu.stratosphere.types.Record$InternalDeSerializer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.