Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.Reader

755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
      }
      if (dump) {
        if (!fs.isFile(logPath)) {
          throw new IOException(args[i] + " is not a file");
        }
        Reader log = new SequenceFile.Reader(fs, logPath, conf);
        try {
          HLogKey key = new HLogKey();
          HLogEdit val = new HLogEdit();
          while (log.next(key, val)) {
            System.out.println(key.toString() + " " + val.toString());
          }
        } finally {
          log.close();
        }
      } else {
        if (!fs.getFileStatus(logPath).isDir()) {
          throw new IOException(args[i] + " is not a directory");
        }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
  public void testAppend() throws IOException {
    final int COL_COUNT = 10;
    final byte [] regionName = Bytes.toBytes("regionname");
    final byte [] tableName = Bytes.toBytes("tablename");
    final byte [] row = Bytes.toBytes("row");
    Reader reader = null;
    HLog log = new HLog(fs, dir, this.conf, null);
    try {
      // Write columns named 1, 2, 3, etc. and then values of single byte
      // 1, 2, 3...
      long timestamp = System.currentTimeMillis();
      TreeMap<HStoreKey, byte []> cols = new TreeMap<HStoreKey, byte []>();
      for (int i = 0; i < COL_COUNT; i++) {
        cols.put(new HStoreKey(row, Bytes.toBytes(Integer.toString(i)), timestamp),
            new byte[] { (byte)(i + '0') });
      }
      log.append(regionName, tableName, cols);
      long logSeqId = log.startCacheFlush();
      log.completeCacheFlush(regionName, tableName, logSeqId);
      log.close();
      Path filename = log.computeFilename(log.getFilenum());
      log = null;
      // Now open a reader on the log and assert append worked.
      reader = new SequenceFile.Reader(fs, filename, conf);
      HLogKey key = new HLogKey();
      HLogEdit val = new HLogEdit();
      for (int i = 0; i < COL_COUNT; i++) {
        reader.next(key, val);
        assertTrue(Bytes.equals(regionName, key.getRegionName()));
        assertTrue(Bytes.equals(tableName, key.getTablename()));
        assertTrue(Bytes.equals(row, key.getRow()));
        assertEquals((byte)(i + '0'), val.getVal()[0]);
        System.out.println(key + " " + val);
      }
      while (reader.next(key, val)) {
        // Assert only one more row... the meta flushed row.
        assertTrue(Bytes.equals(regionName, key.getRegionName()));
        assertTrue(Bytes.equals(tableName, key.getTablename()));
        assertTrue(Bytes.equals(HLog.METAROW, key.getRow()));
        assertTrue(Bytes.equals(HLog.METACOLUMN, val.getColumn()));
        assertEquals(0, HLogEdit.completeCacheFlush.compareTo(val.getVal()));
        System.out.println(key + " " + val);
      }
    } finally {
      if (log != null) {
        log.closeAndDelete();
      }
      if (reader != null) {
        reader.close();
      }
    }
  }
View Full Code Here
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
      }
      if (dump) {
        if (!fs.isFile(logPath)) {
          throw new IOException(args[i] + " is not a file");
        }
        Reader log = new SequenceFile.Reader(fs, logPath, conf);
        try {
          HLogKey key = new HLogKey();
          HLogEdit val = new HLogEdit();
          while (log.next(key, val)) {
            System.out.println(key.toString() + " " + val.toString());
          }
        } finally {
          log.close();
        }
      } else {
        if (!fs.getFileStatus(logPath).isDir()) {
          throw new IOException(args[i] + " is not a directory");
        }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    // import the InMemOutputs
    IntWritable key = new IntWritable();
    MapredOutput value = new MapredOutput();

    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);

      try {
        while (reader.next(key, value)) {
          output.put(key.get(), value.clone());
        }
      } finally {
        reader.close();
      }
    }

    return processOutput(output, callback);
  }
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
    TreeID key = new TreeID();
    MapredOutput value = new MapredOutput();
   
    int index = 0;
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);

      try {
        while (reader.next(key, value)) {
          if (keys != null) {
            keys[index] = key.clone();
          }
         
          if (trees != null) {
            trees[index] = value.getTree();
          }
         
          processOutput(firstIds, key, value, callback);
         
          index++;
        }
      } finally {
        reader.close();
      }
    }

    // make sure we got all the keys/values
    if (index != keys.length) {
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
    TreeID key = new TreeID();
    MapredOutput value = new MapredOutput();
   
    int index = 0;
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, job);

      try {
        while (reader.next(key, value)) {
          if (keys != null) {
            keys[index] = key.clone();
          }
         
          if (trees != null) {
            trees[index] = value.getTree();
          }
         
          processOutput(firstIds, key, value, callback);
         
          index++;
        }
      } finally {
        reader.close();
      }
    }

    // make sure we got all the keys/values
    if (index != keys.length) {
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
    IntWritable key = new IntWritable();
    Step0Output value = new Step0Output(0L, 0);

    int index = 0;
    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, job);

      try {
        while (reader.next(key, value)) {
          keys[index] = key.get();
          values[index] = value.clone();

          index++;
        }
      } finally {
        reader.close();
      }
    }

    return processOutput(keys, values);
  }
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
150
    sorter.merge(outfiles, output);

    // import the descriptions
    LongWritable key = new LongWritable();
    Text value = new Text();
    Reader reader = new Reader(fs, output, conf);

    while (reader.next(key, value)) {
      descriptions.add(value.toString());
    }

    reader.close();
  }
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
    // read all the outputs
    IntWritable key = new IntWritable();
    Step0Output value = new Step0Output(0L, 0);

    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);

      try {
        while (reader.next(key, value)) {
          keys.add(key.get());
          values.add(value.clone());
        }
      } finally {
        reader.close();
      }
    }

    return processOutput(keys, values);
  }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
    // import the InMemOutputs
    IntWritable key = new IntWritable();
    MapredOutput value = new MapredOutput();

    for (Path path : outfiles) {
      Reader reader = new Reader(fs, path, conf);

      try {
        while (reader.next(key, value)) {
          output.put(key.get(), value.clone());
        }
      } finally {
        reader.close();
      }
    }

    return processOutput(output, callback);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Reader

  • com.cloudera.sqoop.testutil.SeqFileReader
  • com.digitalpebble.behemoth.SerializationTest
  • com.m6d.filecrush.crush.Crush
  • com.m6d.filecrush.crush.CrushPartitioner
  • com.m6d.filecrush.crush.CrushReducerParameterizedTest
  • com.m6d.filecrush.crush.CrushStandAloneSequenceFileTest
  • com.m6d.filecrush.crush.CrushTest
  • com.m6d.filecrush.crush.integration.CrushMapReduceTest
  • org.apache.accumulo.server.logger.LogWriter
  • org.apache.accumulo.server.tabletserver.TabletServer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.