Package org.apache.hadoop.typedbytes

Examples of org.apache.hadoop.typedbytes.TypedBytesInput

307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
        }
       
        protected double[] decodeTypedBytesArray(TypedBytesWritable bytes)
            throws IOException {
           
            TypedBytesInput in =
                new TypedBytesInput(
                    new DataInputStream(
                        new ByteArrayInputStream(bytes.getBytes())));
                       
            Type t = in.readType();
            if (t == Type.VECTOR || t == Type.LIST) {
                if (t == Type.VECTOR) {
                    ArrayList<Double> d = new ArrayList<Double>();
                    int len = in.readVectorHeader();
                    for (int i=0; i<len; ++i) {
                        Type et = in.readType();
                        d.add(new Double(readDouble(in, et)));
                    }
                    return doubleArrayListToArray(d);
                } else {
                    ArrayList<Double> d = new ArrayList<Double>();
                    while (true) {
                        Type et = in.readType();
                        if (et == Type.MARKER) {
                            break;
                        }
                        d.add(new Double(readDouble(in, et)));
                    }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
      args[0] = "/typedbytestest";
      int ret = dumptb.run(args);
      assertEquals("Return value != 0.", 0, ret);

      ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
      TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(in));
      int counter = 0;
      Object key = tbinput.read();
      while (key != null) {
        assertEquals(Long.class, key.getClass()); // offset
        Object value = tbinput.read();
        assertEquals(String.class, value.getClass());
        assertTrue("Invalid output.",
          Integer.parseInt(value.toString()) % 10 == 0);
        counter++;
        key = tbinput.read();
      }
      assertEquals("Wrong number of outputs.", 100, counter);
    } finally {
      try {
        fs.close();
View Full Code Here
41
42
43
44
45
46
47
  public void initialize(PipeMapRed pipeMapRed) throws IOException {
    super.initialize(pipeMapRed);
    clientIn = pipeMapRed.getClientInput();
    key = new TypedBytesWritable();
    value = new TypedBytesWritable();
    in = new TypedBytesInput(clientIn);
  }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    FileSystem fs = path.getFileSystem(getConf());
    if (fs.exists(path)) {
      System.err.println("given path exists already!");
      return -1;
    }
    TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path,
      TypedBytesWritable.class, TypedBytesWritable.class);
    try {
      TypedBytesWritable key = new TypedBytesWritable();
      TypedBytesWritable value = new TypedBytesWritable();
      byte[] rawKey = tbinput.readRaw();
      while (rawKey != null) {
        byte[] rawValue = tbinput.readRaw();
        key.set(rawKey, 0, rawKey.length);
        value.set(rawValue, 0, rawValue.length);
        writer.append(key, value);
        rawKey = tbinput.readRaw();
      }
    } finally {
      writer.close();
    }
    return 0;
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    FileSystem fs = path.getFileSystem(getConf());
    if (fs.exists(path)) {
      System.err.println("given path exists already!");
      return -1;
    }
    TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path,
      TypedBytesWritable.class, TypedBytesWritable.class);
    try {
      TypedBytesWritable key = new TypedBytesWritable();
      TypedBytesWritable value = new TypedBytesWritable();
      byte[] rawKey = tbinput.readRaw();
      while (rawKey != null) {
        byte[] rawValue = tbinput.readRaw();
        key.set(rawKey, 0, rawKey.length);
        value.set(rawValue, 0, rawValue.length);
        writer.append(key, value);
        rawKey = tbinput.readRaw();
      }
    } finally {
      writer.close();
    }
    return 0;
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
      args[0] = "/typedbytestest";
      int ret = dumptb.run(args);
      assertEquals("Return value != 0.", 0, ret);

      ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
      TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(in));
      int counter = 0;
      Object key = tbinput.read();
      while (key != null) {
        assertEquals(Long.class, key.getClass()); // offset
        Object value = tbinput.read();
        assertEquals(String.class, value.getClass());
        assertTrue("Invalid output.",
          Integer.parseInt(value.toString()) % 10 == 0);
        counter++;
        key = tbinput.read();
      }
      assertEquals("Wrong number of outputs.", 100, counter);
    } finally {
      try {
        fs.close();
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
  public TypedBytesMapApp(String find) {
    this.find = find;
  }

  public void go() throws IOException {
    TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(System.out));

    Object key = tbinput.readRaw();
    while (key != null) {
      Object value = tbinput.read();
      for (String part : value.toString().split(find)) {
        tboutput.write(part);  // write key
        tboutput.write(1);     // write value
      }
      System.err.println("reporter:counter:UserCounters,InputLines,1");
      key = tbinput.readRaw();
    }
   
    System.out.flush();
  }
View Full Code Here
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
import org.apache.hadoop.typedbytes.TypedBytesOutput;

public class TypedBytesReduceApp {

  public void go() throws IOException {
    TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(System.out));
   
    Object prevKey = null;
    int sum = 0;
    Object key = tbinput.read();
    while (key != null) {
      if (prevKey != null && !key.equals(prevKey)) {
        tboutput.write(prevKey);  // write key
        tboutput.write(sum);      // write value
        sum = 0;
      }
      sum += (Integer) tbinput.read();
      prevKey = key;
      key = tbinput.read();
    }
    tboutput.write(prevKey);
    tboutput.write(sum);
   
    System.out.flush();
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
      args[0] = "/typedbytestest";
      int ret = dumptb.run(args);
      assertEquals("Return value != 0.", 0, ret);

      ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
      TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(in));
      int counter = 0;
      Object key = tbinput.read();
      while (key != null) {
        assertEquals(Long.class, key.getClass()); // offset
        Object value = tbinput.read();
        assertEquals(String.class, value.getClass());
        assertTrue("Invalid output.",
          Integer.parseInt(value.toString()) % 10 == 0);
        counter++;
        key = tbinput.read();
      }
      assertEquals("Wrong number of outputs.", 100, counter);
    } finally {
      try {
        fs.close();
View Full Code Here
41
42
43
44
45
46
47
  public void initialize(PipeMapRed pipeMapRed) throws IOException {
    super.initialize(pipeMapRed);
    clientIn = pipeMapRed.getClientInput();
    key = new TypedBytesWritable();
    value = new TypedBytesWritable();
    in = new TypedBytesInput(clientIn);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.typedbytes.TypedBytesInput

  • gov.sandia.dfgleic.TSQR$TSQRIteration
  • org.apache.hadoop.record.Buffer
  • org.apache.hadoop.streaming.io.TypedBytesOutputReader
  • org.apache.hadoop.streaming.LoadTypedBytes
  • org.apache.hadoop.streaming.TestDumpTypedBytes
  • org.apache.hadoop.streaming.TypedBytesMapApp
  • org.apache.hadoop.streaming.TypedBytesReduceApp
  • java.util.TreeMap
  • java.util.List
  • java.util.ArrayList
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.