Examples of RawSplit

Examples of org.apache.hadoop.mapred.JobClient.RawSplit

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
      System.out.println(estOutSize);
      assertEquals(0, estOutSize);
     
      TaskStatus ts = new MapTaskStatus();
      ts.setOutputSize(singleMapOutputSize);
      RawSplit split = new RawSplit();
      split.setDataLength(singleMapInputSize);
      TaskInProgress tip =
        new TaskInProgress(jid, "", split, jc, jip, 0, 1);
      re.updateWithCompletedTask(ts, tip);
    }
   
    assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
    assertEquals(2* singleMapOutputSize * maps / reduces, re.getEstimatedReduceInputSize());

    //add one more map task with input size as 0
    TaskStatus ts = new MapTaskStatus();
    ts.setOutputSize(singleMapOutputSize);
    RawSplit split = new RawSplit();
    split.setDataLength(0);
    TaskInProgress tip =
      new TaskInProgress(jid, "", split, jc, jip, 0, 1);
    re.updateWithCompletedTask(ts, tip);
   
    long expectedTotalMapOutSize = (singleMapOutputSize*11) *
 
View Full Code Here

Examples of org.apache.hama.bsp.BSPJobClient.RawSplit

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    Configuration conf = new Configuration();

    String[] locations = new String[] { "host6", "host4", "host3" };
    String value = "data";
    RawSplit split = new RawSplit();
    split.setLocations(locations);
    split.setBytes(value.getBytes(), 0, value.getBytes().length);
    split.setDataLength(value.getBytes().length);

    assertEquals(value.getBytes().length, (int) split.getDataLength());

    Map<GroomServerStatus, Integer> taskCountInGroomMap = new HashMap<GroomServerStatus, Integer>(
        20);
    BSPResource[] resources = new BSPResource[0];
    BSPJob job = new BSPJob(new BSPJobID("checkpttest", 1), "/tmp");
View Full Code Here

Examples of org.apache.hama.bsp.BSPJobClient.RawSplit

148
149
150
151
152
153
154
155
156
157
158
        LOG.debug("Cannot start task based on id");
      }
      return new String[0];
    }

    RawSplit rawSplit = taskInProgress.getFileSplit();
    if (rawSplit != null) {
      return rawSplit.getLocations();
    }
    return null;
  }
View Full Code Here

Examples of org.apache.hama.bsp.BSPJobClient.RawSplit

147
148
149
150
151
152
153
154
155
156
157
        LOG.debug("Cannot start task based on id");
      }
      return new String[0];
    }

    RawSplit rawSplit = taskInProgress.getFileSplit();
    if (rawSplit != null) {
      return rawSplit.getLocations();
    }
    return null;
  }
View Full Code Here

Examples of org.apache.hama.bsp.BSPJobClient.RawSplit

147
148
149
150
151
152
153
154
155
156
157
        LOG.debug("Cannot start task based on id");
      }
      return new String[0];
    }

    RawSplit rawSplit = taskInProgress.getFileSplit();
    if (rawSplit != null) {
      return rawSplit.getLocations();
    }
    return null;
  }
View Full Code Here

Examples of org.goldenorb.io.input.RawSplit

102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
      Serializer serializer = factory.getSerializer(splits.get(0).getClass());
      serializer.open(buffer);
      for (int i = 0; i < splits.size(); i++) {
        buffer.reset();
        serializer.serialize(splits.get(i));
        RawSplit rawSplit = new RawSplit();
        rawSplit.setClassName(splits.get(i).getClass().getName());
        rawSplit.setDataLength(splits.get(i).getLength());
        rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
        rawSplit.setLocations(splits.get(i).getLocations());
        rawSplits.add(rawSplit);
      }
    } catch (ClassNotFoundException e) {
      e.printStackTrace();
      throw new RuntimeException(e);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.