Package org.apache.hadoop.mapred.JobClient

Examples of org.apache.hadoop.mapred.JobClient.RawSplit

125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
            factory.getSerializer(splits.get(0).getClass());
          serializer.open(buffer);
          for (int i = 0; i < splits.size(); i++) {
            buffer.reset();
            serializer.serialize(splits.get(i));
            RawSplit rawSplit = new RawSplit();
            rawSplit.setClassName(splits.get(i).getClass().getName());
            rawSplit.setDataLength(splits.get(i).getLength());
            rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
            rawSplit.setLocations(splits.get(i).getLocations());
            rawSplits[i] = rawSplit;
          }

        } else {
          InputSplit[] splits = job.getInputFormat().getSplits(job, 1);
          rawSplits = new RawSplit[splits.length];
          DataOutputBuffer buffer = new DataOutputBuffer();
          for (int i = 0; i < splits.length; i++) {
            buffer.reset();
            splits[i].write(buffer);
            RawSplit rawSplit = new RawSplit();
            rawSplit.setClassName(splits[i].getClass().getName());
            rawSplit.setDataLength(splits[i].getLength());
            rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
            rawSplit.setLocations(splits[i].getLocations());
            rawSplits[i] = rawSplit;
          }
        }
       
        int numReduceTasks = job.getNumReduceTasks();
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
            factory.getSerializer(splits.get(0).getClass());
          serializer.open(buffer);
          for (int i = 0; i < splits.size(); i++) {
            buffer.reset();
            serializer.serialize(splits.get(i));
            RawSplit rawSplit = new RawSplit();
            rawSplit.setClassName(splits.get(i).getClass().getName());
            rawSplit.setDataLength(splits.get(i).getLength());
            rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
            rawSplit.setLocations(splits.get(i).getLocations());
            rawSplits[i] = rawSplit;
          }

        } else {
          InputSplit[] splits = job.getInputFormat().getSplits(job, 1);
          rawSplits = new RawSplit[splits.length];
          DataOutputBuffer buffer = new DataOutputBuffer();
          for (int i = 0; i < splits.length; i++) {
            buffer.reset();
            splits[i].write(buffer);
            RawSplit rawSplit = new RawSplit();
            rawSplit.setClassName(splits[i].getClass().getName());
            rawSplit.setDataLength(splits[i].getLength());
            rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
            rawSplit.setLocations(splits[i].getLocations());
            rawSplits[i] = rawSplit;
          }
        }
       
        int numReduceTasks = job.getNumReduceTasks();
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
      System.out.println(estOutSize);
      assertEquals(0, estOutSize);
     
      TaskStatus ts = new MapTaskStatus();
      ts.setOutputSize(singleMapOutputSize);
      RawSplit split = new RawSplit();
      split.setDataLength(0);
      TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0);
      re.updateWithCompletedTask(ts, tip);
    }
   
    assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
      System.out.println(estOutSize);
      assertEquals(0, estOutSize);
     
      TaskStatus ts = new MapTaskStatus();
      ts.setOutputSize(singleMapOutputSize);
      RawSplit split = new RawSplit();
      split.setDataLength(0);
      TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0);
      re.updateWithCompletedTask(ts, tip);
    }
    assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
    assertEquals(2* singleMapOutputSize * maps / reduces, re.getEstimatedReduceInputSize());
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
      System.out.println(estOutSize);
      assertEquals(0, estOutSize);
     
      TaskStatus ts = new MapTaskStatus();
      ts.setOutputSize(singleMapOutputSize);
      RawSplit split = new RawSplit();
      split.setDataLength(singleMapInputSize);
      TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0);
      re.updateWithCompletedTask(ts, tip);
    }
   
    assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
    assertEquals(2* singleMapOutputSize * maps / reduces, re.getEstimatedReduceInputSize());

    //add one more map task with input size as 0
    TaskStatus ts = new MapTaskStatus();
    ts.setOutputSize(singleMapOutputSize);
    RawSplit split = new RawSplit();
    split.setDataLength(0);
    TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0);
    re.updateWithCompletedTask(ts, tip);
   
    long expectedTotalMapOutSize = (singleMapOutputSize*11) *
      ((maps*singleMapInputSize)+maps)/((singleMapInputSize+1)*10+1);
 
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
            factory.getSerializer(splits.get(0).getClass());
          serializer.open(buffer);
          for (int i = 0; i < splits.size(); i++) {
            buffer.reset();
            serializer.serialize(splits.get(i));
            RawSplit rawSplit = new RawSplit();
            rawSplit.setClassName(splits.get(i).getClass().getName());
            rawSplit.setDataLength(splits.get(i).getLength());
            rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
            rawSplit.setLocations(splits.get(i).getLocations());
            rawSplits[i] = rawSplit;
          }

        } else {
          InputSplit[] splits = job.getInputFormat().getSplits(job, 1);
          rawSplits = new RawSplit[splits.length];
          DataOutputBuffer buffer = new DataOutputBuffer();
          for (int i = 0; i < splits.length; i++) {
            buffer.reset();
            splits[i].write(buffer);
            RawSplit rawSplit = new RawSplit();
            rawSplit.setClassName(splits[i].getClass().getName());
            rawSplit.setDataLength(splits[i].getLength());
            rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
            rawSplit.setLocations(splits[i].getLocations());
            rawSplits[i] = rawSplit;
          }
        }
       
        int numReduceTasks = job.getNumReduceTasks();
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
210
211
212
    if (splits == null || splits.length != numMapTasks) {
      throw new IllegalArgumentException("Input split size mismatch: expected="
          + numMapTasks + ", actual=" + ((splits == null) ? -1 : splits.length));
    }

    RawSplit rawSplits[] = new RawSplit[splits.length];
    for (int i = 0; i < splits.length; i++) {
      try {
        rawSplits[i] = new RawSplit();
        rawSplits[i].setClassName(splits[i].getClass().getName());
        rawSplits[i].setDataLength(splits[i].getLength());
        rawSplits[i].setLocations(splits[i].getLocations());
      } catch (InterruptedException ie) {
        throw new IOException(ie);
View Full Code Here
234
235
236
237
238
239
240
241
242
243
244
    JobID jobid = (JobID) taskAttemptID.getJobID();
    assert (jobid == getJobID());

    // Get splits for the TaskAttempt
    RawSplit split = splits[taskAttemptID.getTaskID().getId()];
    int locality = getClosestLocality(taskTracker, split);

    TaskID taskId = taskAttemptID.getTaskID();
    if (!taskId.isMap()) {
      assert false : "Task " + taskId + " is not MAP :";
View Full Code Here
99
100
101
102
103
104
105
106
107
108
    }

    RawSplit[] createSplits(){
      RawSplit[] splits = new RawSplit[numMapTasks];
      for (int i = 0; i < numMapTasks; i++) {
        splits[i] = new RawSplit();
        splits[i].setLocations(new String[0]);
      }
      return splits;
    }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
      System.out.println(estOutSize);
      assertEquals(0, estOutSize);
     
      TaskStatus ts = new MapTaskStatus();
      ts.setOutputSize(singleMapOutputSize);
      RawSplit split = new RawSplit();
      split.setDataLength(0);
      TaskInProgress tip =
        new TaskInProgress(jid, "", split, jc, jip, 0, 1);
      re.updateWithCompletedTask(ts, tip);
    }
    assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.JobClient.RawSplit

  • org.apache.hadoop.mapred.FakeObjectUtilities$FakeJobInProgress
  • org.apache.hadoop.mapred.LocalJobRunner$Job
  • org.apache.hadoop.mapred.SimulatorJobInProgress
  • org.apache.hadoop.mapred.TestResourceEstimation
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.