Examples of MapWritable


Examples of org.apache.hadoop.io.MapWritable

    KeyValuePair<Writable, Writable> convertedRecord = null;

    Class convertedKeyClass = null;
    Class rawKeyClass = null;
    Class rawValueClass = null;
    MapWritable raw = null;

    while ((rawRecord = peer.readNext()) != null) {
      if (rawKeyClass == null && rawValueClass == null) {
        rawKeyClass = rawRecord.getKey().getClass();
        rawValueClass = rawRecord.getValue().getClass();
      }
      convertedRecord = converter.convertRecord(rawRecord, conf);

      if (convertedRecord == null) {
        throw new IOException("The converted record can't be null.");
      }

      Writable convertedKey = convertedRecord.getKey();
      convertedKeyClass = convertedKey.getClass();

      int index = converter.getPartitionId(convertedRecord, partitioner, conf,
          peer, desiredNum);

      if (!writerCache.containsKey(index)) {
        Path destFile = new Path(partitionDir + "/part-" + index + "/file-"
            + peer.getPeerIndex());
        SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf,
            destFile, convertedKeyClass, MapWritable.class,
            CompressionType.NONE);
        writerCache.put(index, writer);
      }

      raw = new MapWritable();
      raw.put(rawRecord.getKey(), rawRecord.getValue());

      writerCache.get(index).append(convertedKey, raw);
    }

    for (SequenceFile.Writer w : writerCache.values()) {
View Full Code Here

Examples of org.apache.nutch.crawl.MapWritable

    // Its not a tradegy.
    add(urlStr, doc, "encoding", parse.getData().getMeta(ENCODING_KEY),
      false, true, true, false, false);

    // Get metadatas.
    MapWritable mw = datum.getMetaData();
    ParseData pd = parse.getData();

    // Add as stored, indexed, and untokenized but not lowercased.
    add(urlStr, doc, ARCCOLLECTION_KEY,
      getMetadataValue(ARCCOLLECTION_KEY, pd, mw),
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.