Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.Text.readFields()

54
55
56
57
58
59
60
61
62
63
64
    sLogger.info("VOCAB SIZE "+s);
    strings = new ArrayList<String>(s);
    map = new HashMap<String, Integer>();
    Text t = new Text();
    for (int i=0; i<s; i++) {
      t.readFields(in);
      String str = t.toString();
      strings.add(i, str);
      map.put(str, new Integer(i));
    }
  }
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
                    // LOG.info("Got Raw Record");
                    // initialize key / value readers .
                    keyReader.reset(keyData, keyOffset, keyLength);
                    valueReader.reset(valueData, valueOffset, valueLength);
                    keyObject.readFields(keyReader);
                    valueObject.readFields(valueReader);
                    this.spillRecord(keyObject, valueObject);
                }
            };

            // create a local file system
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
  }
 
  public void readFields(DataInput in) throws IOException
  {
    Text t = new Text();
    t.readFields(in);
    String s = t.toString();
    readFields(s);
  }
 
  /*
 
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
    DataInputBuffer valueBuf = new DataInputBuffer();
    long prev = Long.MIN_VALUE;
    while (reader.nextRawKey(keyBuf)) {
      reader.nextRawValue(valueBuf);
      key.readFields(keyBuf);
      value.readFields(valueBuf);
      if (prev != Long.MIN_VALUE) {
        assert(prev <= key.get());
        prev = key.get();
      }
      LOG.info("key = " + key.get() + "; value = " + value);
View Full Code Here
595
596
597
598
599
600
601
602
603
604
605
    FSDataInputStream in = dfs.open(new Path(expectedPath));

    String className = in.readUTF();
    Text message = (Text) ReflectionUtils.newInstance(Class.forName(className),
        config);
    message.readFields(in);

    assertEquals("data", message.toString());

    dfs.delete(new Path("checkpoint"), true);
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
        return in.readDouble();
    }
   
    public Text readString(String tag) throws IOException {
        Text text = new Text();
        text.readFields(in);
        return text;
    }
   
    public ByteArrayOutputStream readBuffer(String tag) throws IOException {
        int len = readInt(tag);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
        return super.compare(b1, s1, l1, b2, s2, l2);
      }
     
      try {
        Text logline1 = new Text();
        logline1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1)));
        String line1 = logline1.toString();
        String[] logColumns1 = line1.split(columnSeparator);
       
        Text logline2 = new Text();
        logline2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2)));
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
        logline1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1)));
        String line1 = logline1.toString();
        String[] logColumns1 = line1.split(columnSeparator);
       
        Text logline2 = new Text();
        logline2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2)));
        String line2 = logline2.toString();
        String[] logColumns2 = line2.split(columnSeparator);
       
        if(logColumns1 == null || logColumns2 == null) {
          return super.compare(b1, s1, l1, b2, s2, l2);
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
        return super.compare(b1, s1, l1, b2, s2, l2);
      }
     
      try {
        Text logline1 = new Text();
        logline1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1)));
        String line1 = logline1.toString();
        String[] logColumns1 = line1.split(columnSeparator);
       
        Text logline2 = new Text();
        logline2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2)));
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
        logline1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1)));
        String line1 = logline1.toString();
        String[] logColumns1 = line1.split(columnSeparator);
       
        Text logline2 = new Text();
        logline2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2)));
        String line2 = logline2.toString();
        String[] logColumns2 = line2.split(columnSeparator);
       
        if (logColumns1 == null || logColumns2 == null) {
          return super.compare(b1, s1, l1, b2, s2, l2);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.