Package org.apache.hadoop.io.MapFile

Examples of org.apache.hadoop.io.MapFile.Writer

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
  public void setUp() throws Exception {
    fs = FileSystem.getLocal(conf);
    Path root = new Path("manyMaps");
    fs.mkdirs(root);
    fs.create(new Path(root, "finished")).close();
    Writer writer = new Writer(conf, fs, "manyMaps/odd", IntWritable.class, BytesWritable.class);
    BytesWritable value = new BytesWritable("someValue".getBytes());
    for (int i = 1; i < 1000; i += 2) {
      writer.append(new IntWritable(i), value);
    }
    writer.close();
    writer = new Writer(conf, fs, "manyMaps/even", IntWritable.class, BytesWritable.class);
    for (int i = 0; i < 1000; i += 2) {
      if (i == 10)
        continue;
      writer.append(new IntWritable(i), value);
    }
    writer.close();
  }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    local.delete(new Path(workdir), true);
    ArrayList<String> dirs = new ArrayList<String>();
    try {
      for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
        String path = workdir + "/" + entry.getKey();
        Writer map = new MapFile.Writer(conf, local, path + "/log1", LogFileKey.class, LogFileValue.class);
        for (KeyValue lfe : entry.getValue()) {
          map.append(lfe.key, lfe.value);
        }
        map.close();
        local.create(new Path(path, "finished")).close();
        dirs.add(path);
      }
      // Recover
      SortedLogRecovery recovery = new SortedLogRecovery();
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    local.delete(new Path(workdir), true);
    ArrayList<String> dirs = new ArrayList<String>();
    try {
      for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
        String path = workdir + "/" + entry.getKey();
        Writer map = new MapFile.Writer(conf, local, path + "/log1", LogFileKey.class, LogFileValue.class);
        for (KeyValue lfe : entry.getValue()) {
          map.append(lfe.key, lfe.value);
        }
        map.close();
        local.create(new Path(path, "finished")).close();
        dirs.add(path);
      }
      // Recover
      SortedLogRecovery recovery = new SortedLogRecovery();
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
    Logger.getRootLogger().setLevel(Level.ERROR);
    fs = FileSystem.getLocal(conf);
    Path root = new Path("manyMaps");
    fs.mkdirs(root);
    fs.create(new Path(root, "finished")).close();
    Writer writer = new Writer(conf, fs, "manyMaps/odd", IntWritable.class, BytesWritable.class);
    BytesWritable value = new BytesWritable("someValue".getBytes());
    for (int i = 1; i < 1000; i += 2) {
      writer.append(new IntWritable(i), value);
    }
    writer.close();
    writer = new Writer(conf, fs, "manyMaps/even", IntWritable.class, BytesWritable.class);
    for (int i = 0; i < 1000; i += 2) {
      if (i == 10)
        continue;
      writer.append(new IntWritable(i), value);
    }
    writer.close();
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
    ArrayList<Path> dirs = new ArrayList<Path>();
    try {
      for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
        String path = workdir + "/" + entry.getKey();
        FileSystem ns = fs.getVolumeByPath(new Path(path)).getFileSystem();
        @SuppressWarnings("deprecation")
        Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class);
        for (KeyValue lfe : entry.getValue()) {
          map.append(lfe.key, lfe.value);
        }
        map.close();
        ns.create(new Path(path, "finished")).close();
        dirs.add(new Path(path));
      }
      // Recover
      SortedLogRecovery recovery = new SortedLogRecovery(fs);
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
    Path root = new Path("file://" + path);
    fs.mkdirs(root);
    fs.create(new Path(root, "finished")).close();
    FileSystem ns = fs.getVolumeByPath(root).getFileSystem();

    @SuppressWarnings("deprecation")
    Writer oddWriter = new Writer(ns.getConf(), ns, new Path(root, "odd").toString(), IntWritable.class, BytesWritable.class);
    BytesWritable value = new BytesWritable("someValue".getBytes());
    for (int i = 1; i < 1000; i += 2) {
      oddWriter.append(new IntWritable(i), value);
    }
    oddWriter.close();

    @SuppressWarnings("deprecation")
    Writer evenWriter = new Writer(ns.getConf(), ns, new Path(root, "even").toString(), IntWritable.class, BytesWritable.class);
    for (int i = 0; i < 1000; i += 2) {
      if (i == 10)
        continue;
      evenWriter.append(new IntWritable(i), value);
    }
    evenWriter.close();
  }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    local.delete(new Path(workdir), true);
    ArrayList<String> dirs = new ArrayList<String>();
    try {
      for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
        String path = workdir + "/" + entry.getKey();
        Writer map = new MapFile.Writer(conf, local, path + "/log1", LogFileKey.class, LogFileValue.class);
        for (KeyValue lfe : entry.getValue()) {
          map.append(lfe.key, lfe.value);
        }
        map.close();
        local.create(new Path(path, "finished")).close();
        dirs.add(path);
      }
      // Recover
      SortedLogRecovery recovery = new SortedLogRecovery();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.MapFile.Writer

  • org.apache.accumulo.server.tabletserver.log.MultiReaderTest
  • org.apache.accumulo.server.tabletserver.log.SortedLogRecoveryTest
  • org.apache.accumulo.tserver.log.MultiReaderTest
  • org.apache.accumulo.tserver.log.SortedLogRecoveryTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.