Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.Reader

885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
      }
      if (dump) {
        if (!fs.isFile(logPath)) {
          throw new IOException(args[i] + " is not a file");
        }
        Reader log = new SequenceFile.Reader(fs, logPath, conf);
        try {
          HLogKey key = new HLogKey();
          HLogEdit val = new HLogEdit();
          while (log.next(key, val)) {
            System.out.println(key.toString() + " " + val.toString());
          }
        } finally {
          log.close();
        }
      } else {
        if (!fs.getFileStatus(logPath).isDir()) {
          throw new IOException(args[i] + " is not a directory");
        }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
  public void testAppend() throws IOException {
    final int COL_COUNT = 10;
    final byte [] regionName = Bytes.toBytes("regionname");
    final byte [] tableName = Bytes.toBytes("tablename");
    final byte [] row = Bytes.toBytes("row");
    Reader reader = null;
    HLog log = new HLog(fs, dir, this.conf, null);
    try {
      // Write columns named 1, 2, 3, etc. and then values of single byte
      // 1, 2, 3...
      long timestamp = System.currentTimeMillis();
      TreeMap<HStoreKey, byte []> cols = new TreeMap<HStoreKey, byte []>();
      for (int i = 0; i < COL_COUNT; i++) {
        cols.put(new HStoreKey(row, Bytes.toBytes(Integer.toString(i)), timestamp),
            new byte[] { (byte)(i + '0') });
      }
      log.append(regionName, tableName, cols);
      long logSeqId = log.startCacheFlush();
      log.completeCacheFlush(regionName, tableName, logSeqId);
      log.close();
      Path filename = log.computeFilename(log.getFilenum());
      log = null;
      // Now open a reader on the log and assert append worked.
      reader = new SequenceFile.Reader(fs, filename, conf);
      HLogKey key = new HLogKey();
      HLogEdit val = new HLogEdit();
      for (int i = 0; i < COL_COUNT; i++) {
        reader.next(key, val);
        assertTrue(Bytes.equals(regionName, key.getRegionName()));
        assertTrue(Bytes.equals(tableName, key.getTablename()));
        assertTrue(Bytes.equals(row, key.getRow()));
        assertEquals((byte)(i + '0'), val.getVal()[0]);
        System.out.println(key + " " + val);
      }
      while (reader.next(key, val)) {
        // Assert only one more row... the meta flushed row.
        assertTrue(Bytes.equals(regionName, key.getRegionName()));
        assertTrue(Bytes.equals(tableName, key.getTablename()));
        assertTrue(Bytes.equals(HLog.METAROW, key.getRow()));
        assertTrue(Bytes.equals(HLog.METACOLUMN, val.getColumn()));
        assertEquals(0, HLogEdit.completeCacheFlush.compareTo(val.getVal()));
        System.out.println(key + " " + val);
      }
    } finally {
      if (log != null) {
        log.closeAndDelete();
      }
      if (reader != null) {
        reader.close();
      }
    }
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
      } else {
        classifier = new StandardNaiveBayesClassifier(model);
      }
      SequenceFile.Writer writer =
          new SequenceFile.Writer(fs, getConf(), getOutputPath(), Text.class, VectorWritable.class);
      Reader reader = new Reader(fs, getInputPath(), getConf());
      Text key = new Text();
      VectorWritable vw = new VectorWritable();
      while (reader.next(key, vw)) {
        writer.append(new Text(SLASH.split(key.toString())[1]),
            new VectorWritable(classifier.classifyFull(vw.get())));
      }
      writer.close();
      reader.close();
    } else {
      boolean succeeded = runMapReduce(parsedArgs);
      if (!succeeded) {
        return -1;
      }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    writer.append(1L, "one");
    writer.append(2L, "two");
   
    writer.close();
   
    Reader reader = new Reader(fs, file, conf);
    assertEquals(1L, reader.next((Object) null));
    assertEquals("one", reader.getCurrentValue((Object) null));
    assertEquals(2L, reader.next((Object) null));
    assertEquals("two", reader.getCurrentValue((Object) null));
    assertNull(reader.next((Object) null));
    reader.close();
   
  }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
      } else {
        classifier = new StandardNaiveBayesClassifier(model);
      }
      SequenceFile.Writer writer =
          new SequenceFile.Writer(fs, getConf(), getOutputPath(), Text.class, VectorWritable.class);
      SequenceFile.Reader reader = new Reader(fs, getInputPath(), getConf());
      Text key = new Text();
      VectorWritable vw = new VectorWritable();
      while (reader.next(key, vw)) {
        writer.append(new Text(key.toString().split("/")[1]),
            new VectorWritable(classifier.classifyFull(vw.get())));
      }
      writer.close();
      reader.close();
    } else {
      boolean succeeded = runMapReduce(parsedArgs);
      if (!succeeded) {
        return -1;
      }
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    List<String> actual = new ArrayList<String>();

    Text text = new Text();
    IntWritable value = new IntWritable();

    Reader reader = new Reader(FileSystem.get(job), new Path(crushOutput.getAbsolutePath()), job);

    while (reader.next(text, value)) {
      actual.add(format("%s\t%d", text, value.get()));
    }

    reader.close();

    int expLines = 0;
    List<List<String>> expected = new ArrayList<List<String>>();

View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
  }

  private void verifyFile(File dir, String fileName, int key, int count) throws IOException {
    File file = new File(dir, fileName);

    Reader reader = new Reader(FileSystem.get(job), new Path(file.getAbsolutePath()), job);

    int i = 0;
    int actual = 0;

    Text text = new Text();
    IntWritable value = new IntWritable();

    while (reader.next(text, value)) {
      assertThat(text.toString(), equalTo(Integer.toString(key)));
      assertThat(value.get(), equalTo(i));

      if (i == 9) {
        i = 0;
      } else {
        i++;
      }

      actual++;
    }

    reader.close();

    assertThat(actual, equalTo(count));
  }
View Full Code Here
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
        fail(crushOutMask);
      }

      Path crushOut = globStatus[0].getPath();

      Reader reader = new Reader(getFileSystem(), crushOut, getFileSystem().getConf());

      assertThat(reader.isBlockCompressed(), is(true));
      assertThat(reader.getCompressionCodec().getClass(), equalTo((Object) codec.getClass()));

      CustomWritable key = new CustomWritable();
      CustomWritable value = new CustomWritable();

      Set<String> expected = new HashSet<String>();
      Set<String> actual = new HashSet<String>();

      for (String fileName : fileNames) {
        int max = Integer.parseInt(fileName.substring(4));

        for (int k = 1, v = max * 100 + 1; k <= max; k++, v++) {
          reader.next(key, value);

          assertThat(expected.add(String.format("%s\t%s", k, v)), is(true));
          assertThat(actual.add(String.format("%s\t%s", key, value)), is(true));
        }
      }

      assertThat("Should be at end of crush output file " + crushOut, reader.next(key, value), is(false));

      reader.close();

      assertThat(actual, equalTo(expected));

    } else if (Format.TEXT == inFmt && Format.SEQUENCE == outFmt) {

      FileStatus[] globStatus = getFileSystem().globStatus(new Path(dir + "/" + crushOutMask));

      if (globStatus == null || 1 != globStatus.length || globStatus[0].isDir()) {
        fail(crushOutMask);
      }

      Path crushOut = globStatus[0].getPath();

      Reader reader = new Reader(getFileSystem(), crushOut, getFileSystem().getConf());

      assertThat(reader.isCompressed(), is(true));

      assertThat(reader.isBlockCompressed(), is(true));
      assertThat(reader.getCompressionCodec().getClass(), equalTo((Object) codec.getClass()));

      Text key = new Text();
      Text value = new Text();

      Set<String> expected = new HashSet<String>();
      Set<String> actual = new HashSet<String>();

      for (String fileName : fileNames) {
        int max = Integer.parseInt(fileName.substring(4));

        for (int k = 1, v = max * 100 + 1; k <= max; k++, v++) {
          reader.next(key, value);

          assertThat(expected.add(String.format("%s\t%s", k, v)), is(true));
          assertThat(actual.add(String.format("%s\t%s", key, value)), is(true));
        }
      }

      assertThat("Should be at end of crush output file " + crushOut, reader.next(key, value), is(false));

      reader.close();

      assertThat(actual, equalTo(expected));

    } else {
      fail();
View Full Code Here
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
    Text prevCrushOut  = new Text();

    for (FileStatus partFile : listStatus) {
      Path path = partFile.getPath();

      Reader reader = new Reader(fs, path, fs.getConf());

      try {
        while (reader.next(srcFile, crushOut)) {
          if (!crushOut.equals(prevCrushOut)) {
            swap(crushInput, prevCrushOut.toString());

            prevCrushOut.set(crushOut);
            crushInput = new LinkedList<Path>();
          }

          crushInput.add(new Path(srcFile.toString()));
        }
      } finally {
        try {
          reader.close();
        } catch (IOException e) {
          LOG.warn("Trapped exception when closing " + path, e);
        }
      }
View Full Code Here
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
    Set<String> crushOutputFiles = new HashSet<String>(nBuckets);

    for (FileStatus partFile : listStatus) {
      Path path = partFile.getPath();

      Reader reader = new Reader(fs, path, fs.getConf());

      try {
        while (reader.next(srcFile, crushOut)) {
          crushOutputFiles.add(new Path(crushOut.toString()).toUri().getPath());
        }
      } finally {
        try {
          reader.close();
        } catch (IOException e) {
          LOG.warn("Trapped exception when closing " + path, e);
        }
      }
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Reader

  • com.cloudera.sqoop.testutil.SeqFileReader
  • com.digitalpebble.behemoth.SerializationTest
  • com.m6d.filecrush.crush.Crush
  • com.m6d.filecrush.crush.CrushPartitioner
  • com.m6d.filecrush.crush.CrushReducerParameterizedTest
  • com.m6d.filecrush.crush.CrushStandAloneSequenceFileTest
  • com.m6d.filecrush.crush.CrushTest
  • com.m6d.filecrush.crush.integration.CrushMapReduceTest
  • org.apache.accumulo.server.logger.LogWriter
  • org.apache.accumulo.server.tabletserver.TabletServer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.