Package org.apache.avro.file

Examples of org.apache.avro.file.CodecFactory

130
131
132
133
134
135
136
137
138
139
140
    dataFileWriter = new DataFileWriter<Object>(writer);

    dataFileWriter.setSyncInterval(syncIntervalBytes);

    try {
      CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
      dataFileWriter.setCodec(codecFactory);
    } catch (AvroRuntimeException e) {
      logger.warn("Unable to instantiate avro codec with name (" +
          compressionCodec + "). Compression disabled. Exception follows.", e);
    }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
    if (org.apache.hadoop.mapred.FileOutputFormat.getCompressOutput(jc)) {
      int level = conf.getInt(org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY,
          org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL);
      String codecName = conf.get(AvroJob.OUTPUT_CODEC,
          org.apache.avro.file.DataFileConstants.DEFLATE_CODEC);
      CodecFactory codec = codecName.equals(org.apache.avro.file.DataFileConstants.DEFLATE_CODEC)
          ? CodecFactory.deflateCodec(level)
          : CodecFactory.fromString(codecName);
      writer.setCodec(codec);
    }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    assertTrue(AvroOutputFormat.getCodecFactory(job) == null);
  }
 
  @Test
  public void testBZip2CodecUsingHadoopClass() throws UnsupportedEncodingException {
    CodecFactory avroBZip2Codec = CodecFactory.fromString("bzip2");
   
    JobConf job = new JobConf();
    job.set("mapred.output.compress", "true");
    job.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.BZip2Codec");
    CodecFactory factory = AvroOutputFormat.getCodecFactory(job);
    assertTrue(factory != null);
    assertTrue(factory.getClass().equals(avroBZip2Codec.getClass()));   
  }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    assertTrue(factory.getClass().equals(avroBZip2Codec.getClass()));   
  }
 
  @Test
  public void testBZip2CodecUsingAvroCodec() throws UnsupportedEncodingException {
    CodecFactory avroBZip2Codec = CodecFactory.fromString("bzip2");
   
    JobConf job = new JobConf();
    job.set("mapred.output.compress", "true");
    job.set(AvroJob.OUTPUT_CODEC, "bzip2");
    CodecFactory factory = AvroOutputFormat.getCodecFactory(job);
    assertTrue(factory != null);
    assertTrue(factory.getClass().equals(avroBZip2Codec.getClass()));   
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
    assertTrue(factory.getClass().equals(avroBZip2Codec.getClass()));   
  }
 
  @Test
  public void testDeflateCodecUsingHadoopClass() throws UnsupportedEncodingException {
    CodecFactory avroDeflateCodec = CodecFactory.fromString("deflate");
   
    JobConf job = new JobConf();
    job.set("mapred.output.compress", "true");
    job.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.DeflateCodec");
    CodecFactory factory = AvroOutputFormat.getCodecFactory(job);
    assertTrue(factory != null);
    assertTrue(factory.getClass().equals(avroDeflateCodec.getClass()));   
  }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    assertTrue(factory.getClass().equals(avroDeflateCodec.getClass()));   
  }
 
  @Test
  public void testDeflateCodecUsingAvroCodec() throws UnsupportedEncodingException {
    CodecFactory avroDeflateCodec = CodecFactory.fromString("deflate");
   
    JobConf job = new JobConf();
    job.set("mapred.output.compress", "true");
    job.set(AvroJob.OUTPUT_CODEC, "deflate");
    CodecFactory factory = AvroOutputFormat.getCodecFactory(job);
    assertTrue(factory != null);
    assertTrue(factory.getClass().equals(avroDeflateCodec.getClass()));   
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
115
    assertTrue(factory.getClass().equals(avroDeflateCodec.getClass()));   
  }
 
  @Test
  public void testSnappyCodecUsingHadoopClass() throws UnsupportedEncodingException {
    CodecFactory avroSnappyCodec = CodecFactory.fromString("snappy");
   
    JobConf job = new JobConf();
    job.set("mapred.output.compress", "true");
    job.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.SnappyCodec");
    CodecFactory factory = AvroOutputFormat.getCodecFactory(job);
    assertTrue(factory != null);
    assertTrue(factory.getClass().equals(avroSnappyCodec.getClass()));   
  }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
126
127
    assertTrue(factory.getClass().equals(avroSnappyCodec.getClass()));   
  }
 
  @Test
  public void testSnappyCodecUsingAvroCodec() throws UnsupportedEncodingException {
    CodecFactory avroSnappyCodec = CodecFactory.fromString("snappy");
   
    JobConf job = new JobConf();
    job.set("mapred.output.compress", "true");
    job.set(AvroJob.OUTPUT_CODEC, "snappy");
    CodecFactory factory = AvroOutputFormat.getCodecFactory(job);
    assertTrue(factory != null);
    assertTrue(factory.getClass().equals(avroSnappyCodec.getClass()));   
  }
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
137
138
139
    assertTrue(factory.getClass().equals(avroSnappyCodec.getClass()));   
  }
 
  @Test
  public void testGZipCodecUsingHadoopClass() throws UnsupportedEncodingException {
    CodecFactory avroDeflateCodec = CodecFactory.fromString("deflate");
   
    JobConf job = new JobConf();
    job.set("mapred.output.compress", "true");
    job.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.GZipCodec");
    CodecFactory factory = AvroOutputFormat.getCodecFactory(job);
    assertTrue(factory != null);
    assertTrue(factory.getClass().equals(avroDeflateCodec.getClass()));   
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
  }
 
  static <T> void configureDataFileWriter(DataFileWriter<T> writer,
      JobConf job) throws UnsupportedEncodingException {
   
    CodecFactory factory = getCodecFactory(job);
   
    if (factory != null) {
      writer.setCodec(factory); 
    }
   
View Full Code Here
TOP

Related Classes of org.apache.avro.file.CodecFactory

  • com.cloudera.flume.handlers.avro.AvroDataFileOutputFormat
  • com.datasalt.pangool.examples.avro.TestAvroTweetsJoin
  • com.datasalt.pangool.tuplemr.avro.AvroOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.linkedin.haivvreo.AvroContainerOutputFormat
  • org.apache.avro.hadoop.file.HadoopCodecFactory
  • org.apache.avro.hadoop.file.TestHadoopCodecFactory
  • org.apache.avro.mapred.AvroOutputFormat
  • org.apache.avro.mapred.TestAvroOutputFormat
  • org.apache.avro.mapreduce.lib.output.AvroOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.