Package org.apache.trevni

Examples of org.apache.trevni.ColumnFileMetaData

69
70
71
72
73
74
75
76
77
78
79
    boolean isMapOnly = job.getNumReduceTasks() == 0;
    final Schema schema = isMapOnly
      ? AvroJob.getMapOutputSchema(job)
      : AvroJob.getOutputSchema(job);

    final ColumnFileMetaData meta = filterMetadata(job);

    final Path dir = FileOutputFormat.getTaskOutputPath(job, name);
    final FileSystem fs = dir.getFileSystem(job);
    if (!fs.mkdirs(dir))
      throw new IOException("Failed to create directory: " + dir);
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
      }
    };
  }

   static ColumnFileMetaData filterMetadata(final JobConf job) {
    final ColumnFileMetaData meta = new ColumnFileMetaData();
    for (Map.Entry<String,String> e : job)
      if (e.getKey().startsWith(META_PREFIX))
        meta.put(e.getKey().substring(META_PREFIX.length()),
                 e.getValue().getBytes(MetaData.UTF8));
    return meta;
  }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
    File outputFile = new File(args.get(2));

    Schema schema = Schema.parse(schemaFile);

    AvroColumnWriter<Object> writer =
      new AvroColumnWriter<Object>(schema, new ColumnFileMetaData());

    for (Object datum : new RandomData(schema, count))
      writer.write(datum);

    writer.writeTo(outputFile);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    Schema schema = Schema.parse(new File(dir, "input.avsc"));
    List<Object> data = fromJson(schema, new File(dir, "input.json"));

    // write full data
    AvroColumnWriter<Object> writer =
      new AvroColumnWriter<Object>(schema, new ColumnFileMetaData());
    for (Object datum : data)
      writer.write(datum);
    writer.writeTo(FILE);

    // test that the full schema reads correctly
View Full Code Here
TOP

Related Classes of org.apache.trevni.ColumnFileMetaData

  • com.linkedin.pig.TrevniStorage
  • com.linkedin.pig.TrevniStorageOutputFormat
  • org.apache.avro.tool.ToTrevniTool
  • org.apache.avro.tool.TrevniCreateRandomTool
  • org.apache.crunch.io.avro.trevni.TrevniFileSourceTargetIT
  • org.apache.crunch.io.avro.trevni.TrevniRecordWriter
  • org.apache.pig.builtin.TrevniStorage
  • org.apache.pig.builtin.TrevniStorageOutputFormat
  • org.apache.tajo.storage.trevni.TrevniAppender
  • org.apache.trevni.avro.AvroTrevniOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.