Package com.cloudera.flume.handlers.avro

Examples of com.cloudera.flume.handlers.avro.AvroJsonOutputFormat

47
48
49
50
51
52
53
54
55
56
57
   */
  @Test
  public void checkSynth() throws IOException, InterruptedException {
    EventSource src = new SynthSource(5, 10, 1337);
    Event e = null;
    EventSink snk = new ConsoleEventSink(new AvroJsonOutputFormat());
    MemorySinkSource mem = new MemorySinkSource();
    while ((e = src.next()) != null) {
      snk.append(e); // visual inspection
      mem.append(e); // testing
    }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
   */
  @Test
  public void checkAttrSynth() throws IOException, InterruptedException {
    EventSource src = new AttrSynthSource(5, 10, 20, 15, 1337);
    Event e = null;
    EventSink snk = new ConsoleEventSink(new AvroJsonOutputFormat());
    MemorySinkSource mem = new MemorySinkSource();
    while ((e = src.next()) != null) {
      snk.append(e); // visual inspection
      mem.append(e); // testing
    }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
    snk.open();
    snk.append(e);
    snk.close();

    ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
    AvroJsonOutputFormat ajof = new AvroJsonOutputFormat();
    ajof.format(exWriter, e);
    exWriter.close();
    String expected = new String(exWriter.toByteArray());

    // check the output to make sure it is what we expected.
    File fo = new File(f.getPath() + "/sub-foo");
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
      }
    };

    String path = "file://" + f + "/%Y-%m-%d/";
    EventSink snk = new HiveNotifyingDfsSink(path, "file-%{host}", "hivetable",
        new AvroJsonOutputFormat(), hfrh);

    snk.open();
    long day_millis = 1000 * 60 * 60 * 24;
    Event e1 = new EventImpl(new byte[0], Clock.unixTime(), Priority.INFO, 0,
        "localhost");
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
    Event e3 = new EventImpl(new byte[0], e1.getTimestamp() + 2 * day_millis,
        Priority.INFO, 0, "localhost");

    String path = "file://" + f + "/%Y-%m-%d/";
    EventSink snk = new HiveNotifyingDfsSink(path, "file-%{host}", "hivetable",
        new AvroJsonOutputFormat(), hfrh);

    snk.open();
    snk.append(e1);
    snk.append(e2);
    snk.append(e3);
    snk.close();

    // Simulate a roll by send the messages a second time sink using the same
    // HiveNewDirNotification handlers.

    // TODO (jon) fix this sink's open close has a problem. here just
    // instantiating a new one
    snk = new HiveNotifyingDfsSink(path, "file-%{host}", "hivetable",
        new AvroJsonOutputFormat(), hfrh);
    snk.open();
    snk.append(e1);
    snk.append(e2);
    snk.append(e3);
    snk.close();
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
    Assert.assertTrue(s.endsWith("hostname test\n"));
  }

  @Test
  public void testAvroJson() throws IOException {
    OutputFormat format = new AvroJsonOutputFormat();
    ByteArrayOutputStream sos = new ByteArrayOutputStream();
    format.format(sos, e);
    String s = new String(sos.toByteArray());
    System.out.print(s);
    // TODO (jon) not sure if this will pass on every machine the same (how does
    // avro order fields?)
    Assert.assertEquals(s, "{\"body\":\"test\",\"timestamp\":0,"
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
   */
  @Test
  public void checkSynth() throws IOException, InterruptedException {
    EventSource src = new SynthSource(5, 10, 1337);
    Event e = null;
    EventSink snk = new ConsoleEventSink(new AvroJsonOutputFormat());
    MemorySinkSource mem = new MemorySinkSource();
    while ((e = src.next()) != null) {
      snk.append(e); // visual inspection
      mem.append(e); // testing
    }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
   */
  @Test
  public void checkAttrSynth() throws IOException, InterruptedException {
    EventSource src = new AttrSynthSource(5, 10, 20, 15, 1337);
    Event e = null;
    EventSink snk = new ConsoleEventSink(new AvroJsonOutputFormat());
    MemorySinkSource mem = new MemorySinkSource();
    while ((e = src.next()) != null) {
      snk.append(e); // visual inspection
      mem.append(e); // testing
    }
View Full Code Here
113
114
115
116
117
118
119
    assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
  }

  @Test
  public void testAvroOutputFormat() throws IOException, InterruptedException {
    checkOutputFormat("avrojson", new AvroJsonOutputFormat());
  }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
    snk.open();
    snk.append(e);
    snk.close();

    ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
    AvroJsonOutputFormat ajof = new AvroJsonOutputFormat();
    ajof.format(exWriter, e);
    exWriter.close();
    String expected = new String(exWriter.toByteArray());

    // check the output to make sure it is what we expected.
    File fo = new File(f.getPath() + "/sub-foo");
View Full Code Here
TOP

Related Classes of com.cloudera.flume.handlers.avro.AvroJsonOutputFormat

  • com.cloudera.flume.core.EventImpl
  • com.cloudera.flume.handlers.debug.TestSynthSources
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.cloudera.flume.handlers.hive.TestHiveNotifyingDfsSink
  • com.cloudera.flume.handlers.text.FormatFactory.OutputFormatBuilder
  • com.cloudera.flume.handlers.text.output.OutputFormat
  • com.cloudera.flume.handlers.text.output.TestOutputFormats
  • com.cloudera.flume.TestSynthSources
  • org.apache.avro.io.JsonEncoder
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.