Package org.apache.hadoop.chukwa

Examples of org.apache.hadoop.chukwa.Chunk

62
63
64
65
66
67
68
69
70
71
72
73
74
      outBuf = new File(BUF_DIR, adaptorID);
      long newOffset = offset;
      if(outBuf.length() > 0) {
        DataInputStream dis = new DataInputStream(new FileInputStream(outBuf));
        while(dis.available() > 0) {
          Chunk c = ChunkImpl.read(dis);
          fSize += c.getData().length;
          long seq = c.getSeqID();
          if(seq >offset) {
            dest.add(c);
            newOffset = seq;
          }
        }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        File outBufTmp = new File(outBuf.getAbsoluteFile(), outBuf.getName() + ".tmp");
        outBuf.renameTo(outBufTmp);
        outToDisk = new DataOutputStream(new FileOutputStream(outBuf, false));
        DataInputStream dis = new DataInputStream(new FileInputStream(outBufTmp));
        while(dis.available() > 0) {
          Chunk c = ChunkImpl.read(dis);
          if(c.getSeqID() > l) { //not yet committed
            c.write(outToDisk);
            fSize += c.getData().length;
          }
        }
        dis.close();
        outBufTmp.delete();
      }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
   
    synchronized void removeUpTo(long l) {

      long bytesFreed = 0;
      while(!chunks.isEmpty()) {
        Chunk c = chunks.getFirst();
        if(c.getSeqID() > l)
          chunks.addFirst(c);
        else
          bytesFreed += c.getData().length;
      }
     
      if(bytesFreed > 0) {
        dataSizeBytes -= bytesFreed;
        notifyAll();
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
    assertEquals(0, agent.adaptorCount());

    String name =agent.processAddCommand("add test = FileAdaptor raw " +testFile.getCanonicalPath() + " 0");
    assertEquals(1, agent.adaptorCount());
    assertEquals(name, "adaptor_test");
    Chunk c = chunks.waitForAChunk(5000);
    assertNotNull(c);
    String dat = new String(c.getData());
    assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
    assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
    assertTrue(c.getDataType().equals("raw"));
    agent.shutdown();
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
     
      assertEquals(0, agent.adaptorCount());
      String name = agent.processAddCommand("add test = FileAdaptor raw " +testFile.getCanonicalPath() + " 0");
      assertEquals(1, agent.adaptorCount());
      assertEquals(name, "adaptor_test");
      Chunk c = chunks.waitForAChunk(5000);
      assertNotNull(c);
      String dat = new String(c.getData());
      assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
      assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
      assertTrue(c.getDataType().equals("raw"));
      if(agent.adaptorCount() > 0)
        agent.stopAdaptor("adaptor_test", false);
    }
    agent.shutdown();
  }
View Full Code Here
38
39
40
41
42
43
44
45
46
    ChunkCatcherConnector chunks = new ChunkCatcherConnector();
    chunks.start();
    String psAgentID = agent.processAddCommand(
        "add org.apache.hadoop.chukwa.datacollection.adaptor.ExecAdaptor ps ps aux 0");
    assertNotNull(psAgentID);
    Chunk c = chunks.waitForAChunk();
    System.out.println(new String(c.getData()));
    agent.shutdown();
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
        this.wait();
      }

      int size = 0;
      while (!queue.isEmpty() && (size < maxSize)) {
        Chunk e = this.queue.remove();
        metrics.removedChunk.inc();
        int chunkSize = e.getData().length;
        size += chunkSize;
        dataSize -= chunkSize;
        metrics.dataSize.set(dataSize);
        events.add(e);
      }
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
   
    public void run() {
      setup();
      try {
        while(sock.isConnected()) {
          Chunk c = sendQ.take();
         
          if(fmt == DataFormat.Raw) {
            byte[] data = c.getData();
            out.writeInt(data.length);
            out.write(data);
          } else if(fmt == DataFormat.Writable)
            c.write(out);
          else {
            byte[] data = c.getData();
            byte[] header = (c.getSource()+ " " + c.getDataType() + " " + c.getStreamName()+ " "+ 
                c.getSeqID()+"\n").getBytes();
            out.writeInt(data.length+ header.length);
            out.write(header);
            out.write(data);
          }
        }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
        this.wait();
      }

      int size = 0;
      while (!queue.isEmpty() && (size < maxSize)) {
        Chunk e = this.queue.remove();
        metrics.removedChunk.inc();
        int chunkSize = e.getData().length;
        size += chunkSize;
        dataSize -= chunkSize;
        metrics.dataSize.set(dataSize);
        events.add(e);
      }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
    int LINES = 50000;
    long bytes = 0;
    long ts_start = System.currentTimeMillis();
    for (int i = 0; i < LINES; ++i) {
      Chunk c = getNewChunk();
      bytes += c.getData().length;
      hlp.process(null, c, nullcollector, Reporter.NULL);
      // hlp.parse(line, nullcollector, Reporter.NULL);
    }
    long time = (System.currentTimeMillis() - ts_start);
    System.out.println("parse took " + time + " milliseconds");
View Full Code Here
TOP

Related Classes of org.apache.hadoop.chukwa.Chunk

  • edu.berkeley.chukwa_xtrace.TestXtrAdaptor
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestCharFileTailingAdaptorUTF8
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailingAdaptorBigRecord
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailingAdaptorPreserveLines
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailingAdaptors
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestLogRotate
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestRawAdaptor
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestRCheckAdaptor
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestStartAtOffset
  • org.apache.hadoop.chukwa.datacollection.adaptor.jms.JMSAdaptor$JMSListener
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.