Package org.apache.hadoop.chukwa.dataloader

Examples of org.apache.hadoop.chukwa.dataloader.DataLoaderFactory

192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
      String[] classes = conf.get(POST_DEMUX_DATA_LOADER,"org.apache.hadoop.chukwa.dataloader.MetricDataLoaderPool,org.apache.hadoop.chukwa.dataloader.FSMDataLoader").split(",");
      for(String dataLoaderName : classes) {
        Class<? extends DataLoaderFactory> dl = (Class<? extends DataLoaderFactory>) Class.forName(dataLoaderName);
        java.lang.reflect.Constructor<? extends DataLoaderFactory> c =
            dl.getConstructor();
        DataLoaderFactory dataloader = c.newInstance();
       
          //DataLoaderFactory dataLoader = (DataLoaderFactory) Class.
          //    forName(dataLoaderName).getConstructor().newInstance();
        log.info(dataLoaderName+" processing: "+directory);
        StringBuilder dirSearch = new StringBuilder();
        dirSearch.append(directory);
        dirSearch.append("/*/*/*.evt");
        Path demuxDir = new Path(dirSearch.toString());
        FileStatus[] events = fs.globStatus(demuxDir);
        dataloader.load(conf, fs, events);
      }
    } catch(Exception e) {
      log.error(ExceptionUtil.getStackTrace(e));
      return false;
    }
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
      String[] classes = conf.get(POST_DEMUX_DATA_LOADER).split(",");
      for(String dataLoaderName : classes) {
        Class<? extends DataLoaderFactory> dl = (Class<? extends DataLoaderFactory>) Class.forName(dataLoaderName);
        java.lang.reflect.Constructor<? extends DataLoaderFactory> c =
            dl.getConstructor();
        DataLoaderFactory dataloader = c.newInstance();
       
          //DataLoaderFactory dataLoader = (DataLoaderFactory) Class.
          //    forName(dataLoaderName).getConstructor().newInstance();
        log.info(dataLoaderName+" processing: "+directory);
        StringBuilder dirSearch = new StringBuilder();
        dirSearch.append(directory);
        dirSearch.append("/*/*/*.evt");
        Path demuxDir = new Path(dirSearch.toString());
        FileStatus[] events = fs.globStatus(demuxDir);
        dataloader.load(conf, fs, events);
      }
    } catch(Exception e) {
      log.error(ExceptionUtil.getStackTrace(e));
      return false;
    }
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
      String[] classes = conf.get(POST_DEMUX_DATA_LOADER,"org.apache.hadoop.chukwa.dataloader.MetricDataLoaderPool,org.apache.hadoop.chukwa.dataloader.FSMDataLoader").split(",");
      for(String dataLoaderName : classes) {
        Class<? extends DataLoaderFactory> dl = (Class<? extends DataLoaderFactory>) Class.forName(dataLoaderName);
        java.lang.reflect.Constructor<? extends DataLoaderFactory> c =
            dl.getConstructor();
        DataLoaderFactory dataloader = c.newInstance();
       
          //DataLoaderFactory dataLoader = (DataLoaderFactory) Class.
          //    forName(dataLoaderName).getConstructor().newInstance();
        log.info(dataLoaderName+" processing: "+directory);
        StringBuilder dirSearch = new StringBuilder();
        dirSearch.append(directory);
        dirSearch.append("/*/*");
        log.debug("dirSearch: " + dirSearch);
        Path demuxDir = new Path(dirSearch.toString());
        // CHUKWA-648:  Make Chukwa Reduce Type to support hierarchy format
        // List all event files under the hierarchy data-type directory
        PathFilter filter = new PathFilter()
        {public boolean accept(Path file) {
          return file.getName().endsWith(".evt");
        }  };
        List<FileStatus> eventfiles = HierarchyDataType.globStatus(fs, demuxDir,filter,true);
        FileStatus[] events = eventfiles.toArray(new FileStatus[eventfiles.size()]);
        dataloader.load(conf, fs, events);
      }
    } catch(Exception e) {
      log.error(ExceptionUtil.getStackTrace(e));
      return false;
    }
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
      String[] classes = conf.get(POST_DEMUX_DATA_LOADER).split(",");
      for(String dataLoaderName : classes) {
        Class<? extends DataLoaderFactory> dl = (Class<? extends DataLoaderFactory>) Class.forName(dataLoaderName);
        java.lang.reflect.Constructor<? extends DataLoaderFactory> c =
            dl.getConstructor();
        DataLoaderFactory dataloader = c.newInstance();
       
          //DataLoaderFactory dataLoader = (DataLoaderFactory) Class.
          //    forName(dataLoaderName).getConstructor().newInstance();
        log.info(dataLoaderName+" processing: "+directory);
        StringBuilder dirSearch = new StringBuilder();
        dirSearch.append(directory);
        dirSearch.append("/*/*");
        log.debug("dirSearch: " + dirSearch);
        Path demuxDir = new Path(dirSearch.toString());
        PathFilter filter = new PathFilter()
        {public boolean accept(Path file) {
          return file.getName().endsWith(".evt");
        }  };
        List<FileStatus> eventfiles = HierarchyDataType.globStatus(fs, demuxDir,filter,true);
        FileStatus[] events = eventfiles.toArray(new FileStatus[eventfiles.size()]);
        dataloader.load(conf, fs, events);
      }
    } catch(Exception e) {
      log.error(ExceptionUtil.getStackTrace(e));
      return false;
    }
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
      String[] classes = conf.get(POST_DEMUX_DATA_LOADER).split(",");
      for(String dataLoaderName : classes) {
        Class<? extends DataLoaderFactory> dl = (Class<? extends DataLoaderFactory>) Class.forName(dataLoaderName);
        java.lang.reflect.Constructor<? extends DataLoaderFactory> c =
            dl.getConstructor();
        DataLoaderFactory dataloader = c.newInstance();
       
          //DataLoaderFactory dataLoader = (DataLoaderFactory) Class.
          //    forName(dataLoaderName).getConstructor().newInstance();
        log.info(dataLoaderName+" processing: "+directory);
        StringBuilder dirSearch = new StringBuilder();
        dirSearch.append(directory);
        dirSearch.append("/*/*/*.evt");
        Path demuxDir = new Path(dirSearch.toString());
        FileStatus[] events = fs.globStatus(demuxDir);
        dataloader.load(conf, fs, events);
      }
    } catch(Exception e) {
      log.error(ExceptionUtil.getStackTrace(e));
      return false;
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.chukwa.dataloader.DataLoaderFactory

  • org.apache.hadoop.chukwa.extraction.demux.PostProcessorManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.