Package hitune.analysis.mapreduce

Examples of hitune.analysis.mapreduce.HiTuneRecord


                Reporter reporter) throws IOException {
            // TODO Auto-generated method stub
            //doing the filter

            String category = this.conf.get(AnalysisProcessorConfiguration.category);
            HiTuneRecord valproxy = new HiTuneRecord(value);
           
            long timestamp = valproxy.getTime();
            String hostname = valproxy.getHost();
           
            if(timestamp >= this.starttime && timestamp <= this.endtime){
                if(isMatched(this.nodelist,hostname)){
                  
                    try {
                        V newvalue = (V) value.getClass().getConstructor().newInstance();
                        HiTuneRecord newvalproxy = new HiTuneRecord(newvalue);
                        newvalproxy.copyCommonFields(value);
                       
                                       
                       
                        for(String metric : this.metrics){
                            String val = valproxy.getValue(metric);
                            if(val == null){
                                val = "";
                            }
                            newvalproxy.add(metric,val);
                            if(category!=null){
                                newvalproxy.add(AnalysisProcessorConfiguration.category,category);
                            }
                        }
                        output.collect(key, (V) newvalproxy.getObject());
                    } catch (IllegalArgumentException e) {
                        // TODO Auto-generated catch block
                        log.warn(e);
                        e.printStackTrace();
                    } catch (SecurityException e) {
View Full Code Here


            // TODO Auto-generated method stub
            //doing the filter

            //<key,value>
            //<[AttemptID/PhaseAlias/ThreadName/ThreadId/Func], [Callee,isLast]>
            HiTuneRecord valproxy = new HiTuneRecord(value);
            HiTuneKey keyproxy = new HiTuneKey(key);

            String hostname = valproxy.getHost();
            String status  = valproxy.getValue("ThreadState");
            String stack = valproxy.getValue("CallStack");
            if(stack !=null && stack.length()!=0)stack = stack.replace(" ", "");
            else stack = "";
            String attemptID = valproxy.getValue("TaskID");
            log.debug("hostname:" + hostname + " ThreadState:" + status + " stack:" + stack + " attemptID:" + attemptID);
            if(isMatched(this.nodelist,hostname)){
                if(isMatched(this.statuslist, status)){
                    for(String s : phasealias.keySet()){
                        log.debug("phasealias:" +s);

                        String phase_name = phasealias.get(s);
                        if(s==null || s.length()==0)s="";
                        Pattern p = Pattern.compile(s);
                        if(stack!=null && stack.length()!=0)stack=stack.replace(" ", "");
                        else stack="";
                        Matcher matcher = p.matcher(stack);
                        if(matcher.find()){


                            String thread_id = valproxy.getValue("ThreadID");
                            String thread_name = valproxy.getValue("ThreadName");


                            try {
                                K newkey = (K) key.getClass().getConstructor().newInstance();
                                V newvalue = (V) value.getClass().getConstructor().newInstance();

                                HiTuneRecord newvalproxy = new HiTuneRecord(newvalue);
                                HiTuneKey newkeyproxy = new HiTuneKey(newkey);

                                String[] fcs = stack.split("#");   

                                String[] funcs = new String [fcs.length+2];
                                funcs[0]="_PHASE_";                         
                                funcs[1]=getFuncPattern(stack, this.phases.get(s));

                                System.arraycopy(fcs,0,funcs,2,fcs.length);
                                for (int i =0; i< funcs.length; i++){

                                    newkeyproxy.setKey(attemptID + "/" + phase_name + "/" + thread_name + "/" + thread_id + "/" + funcs[i]);
                                    newkeyproxy.setDataType(keyproxy.getDataType());
                                    newvalproxy.copyCommonFields(value);
                                    newvalproxy.add("func", funcs[i]);
                                    newvalproxy.add("thread_id", thread_id);
                                    newvalproxy.add("thread_name", thread_name);
                                    newvalproxy.add("phase_name",phase_name );
                                    newvalproxy.add("phase_stack", s);
                                    newvalproxy.add("attempt_id", attemptID);
                                    newvalproxy.add("Callee", "1");
                                    if(i==2){
                                        newvalproxy.add("isLast", "1");
                                    }else{
                                        newvalproxy.add("isLast", "0");
                                    }
                                    output.collect((K)newkeyproxy.getObject(), (V)newvalproxy.getObject());
                                }
                            } catch (IllegalArgumentException e) {
                                // TODO Auto-generated catch block
                                e.printStackTrace();
                                log.warn(e);
View Full Code Here

            long start = -1, end = -1;
            long phaseCount=0;
            String funcCount="", statusCount="";
            String funcList="", statusList="";
            while(values.hasNext()){
                HiTuneRecord valproxy = new HiTuneRecord(values.next());
                long phaseStart = Long.parseLong(valproxy.getValue("start"));
                long phaseEnd = Long.parseLong(valproxy.getValue("start"));
                start = start == -1 ? phaseStart : Math.min(start,phaseStart);
                end = end == -1 ?phaseEnd : Math.max(end,phaseEnd);
                phaseCount++;
                funcCount = funcCount == "" ? valproxy.getValue("funCount"):vectorAdd(valproxy.getValue("funCount"),funcCount, SEPERATOR_COMMA);
                statusCount = statusCount == "" ? valproxy.getValue("statusCount"):vectorAdd(valproxy.getValue("statusCount"),statusCount, SEPERATOR_COMMA);
                newRecord.put("host", valproxy.getHost());
                newRecord.put("job_id", valproxy.getValue(AnalysisProcessorConfiguration.jobid));
                newRecord.put("phase_stack", valproxy.getValue("phase_stack"));
                newRecord.put("phase_name", valproxy.getValue("phase_name"));
                newRecord.put("attempt_id", valproxy.getValue("attempt_id"));
                newRecord.put("thread_id", valproxy.getValue("thread_id"));
                newRecord.put("thread_name", valproxy.getValue("thread_name"));
                funcList = valproxy.getValue("funList");
                statusList = valproxy.getValue("statusList");
            }

            newRecord.put("phase_start", ""+start);
            newRecord.put("phase_end", ""+end);
            newRecord.put("phase_count", ""+phaseCount);
View Full Code Here

                Reporter reporter) throws IOException {
            // TODO Auto-generated method stub
            //doing the filter

            HiTuneKey keyproxy = new HiTuneKey(key);
            HiTuneRecord valproxy = new HiTuneRecord(value);
           
           
            String device = null;       
            long timestamp = valproxy.getTime();
            String hostname = valproxy.getHost();
            String datatype = keyproxy.getDataType();
           
            log.debug("timestamp:" + timestamp + " hostname:" + hostname + " datatype:" + datatype);
            if(timestamp >= this.starttime && timestamp <= this.endtime){
//              if(datatype.equals("HiTune.Cpustat")){
//              device = value.getValue("CPU");
//              }
                if(datatype.equals("HiTune.Iostat")){
                    device = valproxy.getValue("Device:");
                }
                else if(datatype.equals("HiTune.Netstat")){
                    device = valproxy.getValue("IFACE");
                }
                else if(datatype.equals("HiTune.Mpstat")){
                    device = valproxy.getValue("CPU");
                }
                else if(datatype.equals("HiTune.Swtichstat")){
                    device = valproxy.getValue("Port");
                }
                if(isMatched(this.nodelist,hostname)){
                    log.debug("device:" + device);
                    if(device ==null || (device != null && isMatched(this.devices,device))){
                      
                        try {
                            V newvalue = (V) value.getClass().getConstructor().newInstance();
                            HiTuneRecord newvalproxy = new HiTuneRecord(newvalue);
                            newvalproxy.copyCommonFields(value);
                           

                            for(String metric : this.metrics){
                                String val = valproxy.getValue(metric);
                                if(val == null){
                                    val = "";
                                }
                                newvalproxy.add(metric,val);
                            }

                            if(device !=null){
                                newvalproxy.add("device", device);
                            }
                            output.collect(key, (V) newvalproxy.getObject());
                        } catch (IllegalArgumentException e) {
                            // TODO Auto-generated catch block
                            e.printStackTrace();
                            log.warn(e);
                        } catch (SecurityException e) {
View Full Code Here

        throws IOException {
            // TODO Auto-generated method stub
            //organizing into csv format
            while(values.hasNext()){
                V value = (V) values.next();
                HiTuneRecord valproxy = new HiTuneRecord(value);
               
                Text newKey = new Text();
                TextArrayWritable newValue = null;
                String timestamp = "" + valproxy.getTime()/1000;

                String [] fields = valproxy.getFields();
                if(!initialized){
                    newKey.set("Timestamp");
                    newValue = new TextArrayWritable(fields);
                    output.collect(newKey, newValue);
                    initialized = true;
                }
                newKey.set(timestamp);
                String [] contents = new String[fields.length];
                for (int i=0; i<fields.length; i++  ){
                    contents[i] = valproxy.getValue(fields[i]);
                }
                newValue = new TextArrayWritable(contents);
                output.collect(newKey, newValue);
            }
        }
View Full Code Here

            long callee_num = 0;
            long lastlevel_callee_num = 0;

            try{
                V val = null;
                HiTuneRecord valproxy = null;
                K newkey = (K) key.getClass().getConstructor().newInstance();
                HiTuneKey newkeyproxy = new HiTuneKey(newkey);
                while(values.hasNext()){
                    val = (V)values.next();
                    valproxy = new HiTuneRecord(val);
                    callee_num += Integer.parseInt(valproxy.getValue("Callee"));
                    lastlevel_callee_num += Integer.parseInt(valproxy.getValue("isLast"));
                }

                V newvalue = (V) val.getClass().getConstructor().newInstance();
                HiTuneRecord newvalproxy = new HiTuneRecord(newvalue);
                newvalproxy.copyCommonFields(val);

                newvalproxy.add("callee_num", ""+callee_num);
                newvalproxy.add("last_level_callee_num", ""+lastlevel_callee_num );
                newvalproxy.add("attempt_id",valproxy.getValue("attempt_id") );
                newvalproxy.add("phase_name",valproxy.getValue("phase_name") );
                newvalproxy.add("phase_stack", valproxy.getValue("phase_stack") );
                newvalproxy.add("thread_name",valproxy.getValue("thread_name") );
                newvalproxy.add("thread_id",valproxy.getValue("thread_id") );
                newvalproxy.add("host", valproxy.getHost());
                newvalproxy.add("func", valproxy.getValue("func") );


                newkeyproxy.setKey(valproxy.getValue("attempt_id")
                        + "/" + valproxy.getValue("phase_name")
                        + "/" + valproxy.getValue("thread_name")
                        + "/" + valproxy.getValue("thread_id"));
                newkeyproxy.setDataType(new HiTuneKey(key).getDataType());

                output.collect((K)newkeyproxy.getObject(), (V)newvalproxy.getObject());
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (SecurityException e) {
                // TODO Auto-generated catch block
View Full Code Here


            TreeSet <HiTuneRecord> arrays = new TreeSet<HiTuneRecord>(new RecordComparator());

            TreeSet <HiTuneRecord> stackarrays = new TreeSet<HiTuneRecord>(new RecordComparator());
            HiTuneRecord phase = null;
            //log.debug("key: " + key.toString());
            while(values.hasNext()){
                try{
                    HiTuneRecord temp_proxyval = new HiTuneRecord(values.next());

                    V newvalue = (V) temp_proxyval.getObject().getClass().getConstructor().newInstance();
                    HiTuneRecord proxyval = new HiTuneRecord(newvalue);

                    for (String field: temp_proxyval.getFields()){
                        proxyval.add(field, temp_proxyval.getValue(field));
                    }

                    String function = proxyval.getValue("func");
                    log.debug(" val: " + proxyval.toString());

                    if (function.equals("_PHASE_")){
                        phase = proxyval;
                        continue;
                    }else{
                        if(function.indexOf("#")!=-1 ){
                            if(funcInStackFormat){
                                stackarrays.add(proxyval);
                                if(stackarrays.size()> limitNum){
                                    stackarrays.remove(stackarrays.last());
                                }
                            }
                        }else{
                            //log.debug("add new val: " + val);
                            arrays.add(proxyval);
                            if(arrays.size()> limitNum){
                                arrays.remove(arrays.last());
                            }
                        }

                    }

                } catch (IllegalArgumentException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (SecurityException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (InstantiationException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (IllegalAccessException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (InvocationTargetException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (NoSuchMethodException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }

            }

            if(funcInStackFormat){
                int len = limitNum<stackarrays.size()?limitNum:stackarrays.size();
                HiTuneRecord[] candidates = stackarrays.toArray(new HiTuneRecord[0]);
                for (int i = 0; i <stackarrays.size(); i++ ){
                    HiTuneRecord val = candidates[i];
                    if(val!=null){
                        newRecord.clear();
                        for(String head : headers){
                            if(head.equals("phase_count")){
                                newRecord.put(head,  phase.getValue("callee_num"));
                            }else{
                                newRecord.put(head, val.getValue(head));
                            }

                        }
                        String [] contents = new String[newRecord.keySet().size()];
                        int j = 0;
                        for (String index:  newRecord.keySet() ){
                            contents[j] = newRecord.get(index);
                            log.debug("content: " + index + "," +contents[j] );
                            j++;

                        }
                        TextArrayWritable newValue = new TextArrayWritable(contents);
                        output.collect(null, newValue);
                    }


                }
            }else {
                int len = limitNum<arrays.size()?limitNum:arrays.size();
                HiTuneRecord[] candidates = arrays.toArray(new HiTuneRecord[0]);

                for (int i = 0; i <len; i++ ){

                    HiTuneRecord val = candidates[i];
                    log.debug("dump val: " + val);
                    if(val!=null){
                        newRecord.clear();
                        for(String head : headers){
                            if(head.equals("phase_count")){
                                newRecord.put(head,  phase.getValue("callee_num"));
                            }else{
                                newRecord.put(head, val.getValue(head));
                            }
                        }
                        String [] contents = new String[newRecord.keySet().size()];
                        int j = 0;
                        for (String index:  newRecord.keySet() ){
View Full Code Here

                OutputCollector<K, V> output,
                Reporter reporter) throws IOException {
            // TODO Auto-generated method stub
            //doing the filter
            // Get the records of same job id
            HiTuneRecord valproxy = new HiTuneRecord(value);
            HiTuneKey keyproxy = new HiTuneKey(key);

            log.debug(key.toString());
            log.debug(value.toString());
            try {
                K newkey = (K) key.getClass().getConstructor().newInstance();
                HiTuneKey newkeyproxy = new HiTuneKey(newkey);
                String []parts = keyproxy.getDataType().split("/");
                String recordType = parts[parts.length-1];
                log.debug("record_type: " + recordType);
                if(recordType.equals("Job")){
                    newkeyproxy.setKey(valproxy.getValue("JOBID"));
                }
                else if(recordType.equals("MapAttempt")){
                    newkeyproxy.setKey(valproxy.getValue("TASK_ATTEMPT_ID"));
                }
                else if(recordType.equals("MapTask")){
                    newkeyproxy.setKey(valproxy.getValue("TASKID"));
                }
                else if(recordType.equals("ReduceAttempt")){
                    newkeyproxy.setKey(valproxy.getValue("TASK_ATTEMPT_ID"));
                }
                else if(recordType.equals("ReduceTask")){
                    newkeyproxy.setKey(valproxy.getValue("TASKID"));
                }
                if (newkeyproxy.getKey()!=null && newkeyproxy.getKey().length()!=0){
                    newkeyproxy.setDataType(keyproxy.getDataType());
                    output.collect((K) newkeyproxy.getObject(),(V) value);
                }
View Full Code Here

            Text newKey = null;
            TextArrayWritable newValue = null;

            Map<String, String> map = new HashMap<String, String>();
            Map<String, String> filter = new HashMap<String, String>();
            HiTuneRecord valproxy = null;
            while(values.hasNext()){
                valproxy= new HiTuneRecord(values.next());
                //copyGernetic(record, newrecord);
                for(String metric : valproxy.getFields()){
                    log.debug("copy metric:" + metric);
                    String val = valproxy.getValue(metric);
                    if(val == null){
                        val = "";
                    }
                    map.put(metric,val);
                }
            }
            if(valproxy!=null){
                //do filtering
                filter.putAll(valproxy.getCommonFields());

                for(String metric : this.metrics){
                    log.debug("collect metric:" + metric);
                    String val = map.get(metric);
                    if(val == null){
View Full Code Here

            // TODO Auto-generated method stub
            //doing the filter

            //<key,value>
            //<[AttemptID/PhaseStack/PhaseAlias], [ThreadName,ThreadId,starttime,endtime,funlist,funcountlist,statelist,statecountlist,funStateMatric]>
            HiTuneRecord valproxy = new HiTuneRecord(value);
            String hostname = valproxy.getHost();
            String status  = valproxy.getValue("ThreadState");
            String stack = valproxy.getValue("CallStack");
            String attemptID = valproxy.getValue("TaskID");
            log.debug("hostname:" + hostname + " ThreadState:" + status + " stack:" + stack + " attemptID:" + attemptID);
            if(isMatched(this.nodelist,hostname)){
                for(String s : phasealias.keySet()){
                    log.debug("phasealias:" +s);
                    if(s==null || s.length()==0)s="";
                    Pattern p = Pattern.compile(s);
                    if(stack!=null && stack.length()!=0)stack=stack.replace(" ", "");
                    else stack="";
                    Matcher matcher = p.matcher(stack);
                    if(matcher.find()){
                        try{
                            log.debug("find pattern");
                            K newkey = (K) key.getClass().getConstructor().newInstance();
                            V newval = (V) value.getClass().getConstructor().newInstance();

                            HiTuneKey newkeyproxy = new HiTuneKey(newkey);
                            HiTuneRecord newvalproxy = new HiTuneRecord(newval);

                            newkeyproxy.setKey(attemptID + "/" + s + "/" + phasealias.get(s));
                            newkeyproxy.setDataType(new HiTuneKey(key).getDataType());
                            newvalproxy.copyCommonFields(value);


                            newvalproxy.add("thread_id", valproxy.getValue("ThreadID"));
                            newvalproxy.add("thread_name", valproxy.getValue("ThreadName"));
                            newvalproxy.add("attempt_id", attemptID);
                            newvalproxy.add("phase_stack", s);
                            newvalproxy.add("phase_name", phasealias.get(s));
                            newvalproxy.add("start", "" + newvalproxy.getTime());
                            newvalproxy.add("count" , "1");
                            log.debug("status:" + conf.get("status"));
                            newvalproxy.add("statusList", conf.get("status"));
                            newvalproxy.add("statusCount", count(status, this.statuslist));

                            log.debug("funList:" + this.phases.get(s));
                            newvalproxy.add("funList", List2String(this.phases.get(s),SEPERATOR_COMMA));
                            newvalproxy.add("funCount", count(stack, this.phases.get(s)));
                            newvalproxy.add(AnalysisProcessorConfiguration.jobid, conf.get(AnalysisProcessorConfiguration.jobid));

                            log.debug("Key:" + newkeyproxy.toString() + " Record" + newkeyproxy.toString());
                            output.collect((K)newkeyproxy.getObject(), (V)newvalproxy.getObject());
                        } catch (IllegalArgumentException e) {
                            // TODO Auto-generated catch block
                            log.warn(e);
                            e.printStackTrace();
                        } catch (SecurityException e) {
View Full Code Here

TOP

Related Classes of hitune.analysis.mapreduce.HiTuneRecord

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.