Examples of org.apache.tajo.master.querymaster.QueryUnit$InitialScheduleTransition

Package org.apache.tajo.master.querymaster

Examples of org.apache.tajo.master.querymaster.QueryUnit$InitialScheduleTransition

org.apache.tajo.master.querymaster.QueryUnit

        }


        SubQuery subQuery = context.getQuery().getSubQuery(attemptId.getQueryUnitId().getExecutionBlockId());


        if (attemptId != null) {
          QueryUnit task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
          QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
              attemptId,
              new ArrayList<Fragment>(task.getAllFragments()),
              "",
              false,
              task.getLogicalPlan().toJson(),
              context.getQueryContext(),
              subQuery.getDataChannel(), subQuery.getBlock().getEnforcer());
          if (!subQuery.getBlock().isRoot()) {
            taskAssign.setInterQuery();
          }

View Full Code Here

        if (nonLeafTasks.size() > 0) {
          attemptId = nonLeafTasks.iterator().next();
          nonLeafTasks.remove(attemptId);
          LOG.debug("Assigned based on * match");


          QueryUnit task;
          SubQuery subQuery = context.getSubQuery(attemptId.getQueryUnitId().getExecutionBlockId());
          task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
          QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
              attemptId,
              Lists.newArrayList(task.getAllFragments()),
              "",
              false,
              task.getLogicalPlan().toJson(),
              context.getQueryContext(),
              subQuery.getDataChannel(),
              subQuery.getBlock().getEnforcer());
          if (!subQuery.getBlock().isRoot()) {
            taskAssign.setInterQuery();
          }
          for (ScanNode scan : task.getScanNodes()) {
            Collection<URI> fetches = task.getFetch(scan);
            if (fetches != null) {
              for (URI fetch : fetches) {
                taskAssign.addFetch(scan.getTableName(), fetch);
              }
            }

View Full Code Here

      }


      String host = container.getTaskHostName();
      QueryUnitAttemptScheduleContext queryUnitContext = new QueryUnitAttemptScheduleContext(container.containerID,
          host, taskRequest.getCallback());
      QueryUnit task = SubQuery.newEmptyQueryUnit(context, queryUnitContext, subQuery, nextTaskId++);


      FragmentPair fragmentPair;
      List<FragmentPair> fragmentPairs = new ArrayList<FragmentPair>();
      boolean diskLocal = false;
      long assignedFragmentSize = 0;
      long taskSize = adjustTaskSize();
      LOG.info("Adjusted task size: " + taskSize);


      // host local, disk local
      String normalized = NetUtils.normalizeHost(host);
      Integer diskId = hostDiskBalancerMap.get(normalized).getDiskId(container.containerID);
      if (diskId != null && diskId != -1) {
        do {
          fragmentPair = scheduledFragments.getHostLocalFragment(host, diskId);
          if (fragmentPair == null || fragmentPair.getLeftFragment() == null) {
            break;
          }


          if (assignedFragmentSize + fragmentPair.getLeftFragment().getEndKey() > taskSize) {
            break;
          } else {
            fragmentPairs.add(fragmentPair);
            assignedFragmentSize += fragmentPair.getLeftFragment().getEndKey();
            if (fragmentPair.getRightFragment() != null) {
              assignedFragmentSize += fragmentPair.getRightFragment().getEndKey();
            }
          }
          scheduledFragments.removeFragment(fragmentPair);
          diskLocal = true;
        } while (scheduledFragments.size() > 0 && assignedFragmentSize < taskSize);
      }


      if (assignedFragmentSize < taskSize) {
        // host local
        do {
          fragmentPair = scheduledFragments.getHostLocalFragment(host);
          if (fragmentPair == null || fragmentPair.getLeftFragment() == null) {
            break;
          }


          if (assignedFragmentSize + fragmentPair.getLeftFragment().getEndKey() > taskSize) {
            break;
          } else {
            fragmentPairs.add(fragmentPair);
            assignedFragmentSize += fragmentPair.getLeftFragment().getEndKey();
            if (fragmentPair.getRightFragment() != null) {
              assignedFragmentSize += fragmentPair.getRightFragment().getEndKey();
            }
          }
          scheduledFragments.removeFragment(fragmentPair);
          diskLocal = false;
        } while (scheduledFragments.size() > 0 && assignedFragmentSize < taskSize);
      }


      // rack local
      if (fragmentPairs.size() == 0) {
        fragmentPair = scheduledFragments.getRackLocalFragment(host);


        // random
        if (fragmentPair == null) {
          fragmentPair = scheduledFragments.getRandomFragment();
        } else {
          rackLocalAssigned++;
        }


        if (fragmentPair != null) {
          fragmentPairs.add(fragmentPair);
          scheduledFragments.removeFragment(fragmentPair);
        }
      } else {
        if (diskLocal) {
          diskLocalAssigned++;
        } else {
          hostLocalAssigned++;
        }
      }


      if (fragmentPairs.size() == 0) {
        throw new RuntimeException("Illegal State!!!!!!!!!!!!!!!!!!!!!");
      }


      LOG.info("host: " + host + " disk id: " + diskId + " fragment num: " + fragmentPairs.size());


      task.setFragment(fragmentPairs.toArray(new FragmentPair[fragmentPairs.size()]));
      subQuery.getEventHandler().handle(new TaskEvent(task.getId(), TaskEventType.T_SCHEDULE));
    }
  }

View Full Code Here

        LOG.debug("Assigned based on * match");
        ContainerProxy container = context.getMasterContext().getResourceAllocator().getContainer(
            taskRequest.getContainerId());
        QueryUnitAttemptScheduleContext queryUnitContext = new QueryUnitAttemptScheduleContext(container.containerID,
            container.getTaskHostName(), taskRequest.getCallback());
        QueryUnit task = SubQuery.newEmptyQueryUnit(context, queryUnitContext, subQuery, nextTaskId++);
        task.setFragment(scheduledFragments.getAllFragments());
        subQuery.getEventHandler().handle(new TaskEvent(task.getId(), TaskEventType.T_SCHEDULE));
      }
    }
  }

View Full Code Here

            }
          }
        }


        if (attemptId != null) {
          QueryUnit task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
          QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
              attemptId,
              new ArrayList<FragmentProto>(task.getAllFragments()),
              "",
              false,
              task.getLogicalPlan().toJson(),
              context.getMasterContext().getQueryContext(),
              subQuery.getDataChannel(), subQuery.getBlock().getEnforcer());
          if (checkIfInterQuery(subQuery.getMasterPlan(), subQuery.getBlock())) {
            taskAssign.setInterQuery();
          }


          context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId,
              taskRequest.getContainerId(),
              host, container.getTaskPort()));
          assignedRequest.add(attemptId);


          scheduledObjectNum -= task.getAllFragments().size();
          taskRequest.getCallback().run(taskAssign.getProto());
        } else {
          throw new RuntimeException("Illegal State!!!!!!!!!!!!!!!!!!!!!");
        }
      }

View Full Code Here

            attemptId = nonLeafTasks.iterator().next();
            nonLeafTasks.remove(attemptId);
          }
          LOG.debug("Assigned based on * match");


          QueryUnit task;
          task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
          QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
              attemptId,
              Lists.newArrayList(task.getAllFragments()),
              "",
              false,
              task.getLogicalPlan().toJson(),
              context.getMasterContext().getQueryContext(),
              subQuery.getDataChannel(),
              subQuery.getBlock().getEnforcer());
          if (checkIfInterQuery(subQuery.getMasterPlan(), subQuery.getBlock())) {
            taskAssign.setInterQuery();
          }
          for (ScanNode scan : task.getScanNodes()) {
            Collection<URI> fetches = task.getFetch(scan);
            if (fetches != null) {
              for (URI fetch : fetches) {
                taskAssign.addFetch(scan.getTableName(), fetch);
              }
            }

View Full Code Here

    if (event.getType() == EventType.T_SCHEDULE) {
      if (event instanceof FragmentScheduleEvent) {
        FragmentScheduleEvent castEvent = (FragmentScheduleEvent) event;
        if (context.isLeafQuery()) {
          QueryUnitAttemptScheduleContext queryUnitContext = new QueryUnitAttemptScheduleContext();
          QueryUnit task = SubQuery.newEmptyQueryUnit(context, queryUnitContext, subQuery, nextTaskId++);
          task.addFragment(castEvent.getLeftFragment(), true);
          scheduledObjectNum++;
          if (castEvent.hasRightFragments()) {
            task.addFragments(castEvent.getRightFragments());
            //scheduledObjectNum += castEvent.getRightFragments().size();
          }
          subQuery.getEventHandler().handle(new TaskEvent(task.getId(), TaskEventType.T_SCHEDULE));
        } else {
          fragmentsForNonLeafTask = new FileFragment[2];
          fragmentsForNonLeafTask[0] = castEvent.getLeftFragment();
          if (castEvent.hasRightFragments()) {
            fragmentsForNonLeafTask[1] = castEvent.getRightFragments().toArray(new FileFragment[]{})[0];
          }
        }
      } else if (event instanceof FetchScheduleEvent) {
        FetchScheduleEvent castEvent = (FetchScheduleEvent) event;
        Map<String, List<URI>> fetches = castEvent.getFetches();
        QueryUnitAttemptScheduleContext queryUnitContext = new QueryUnitAttemptScheduleContext();
        QueryUnit task = SubQuery.newEmptyQueryUnit(context, queryUnitContext, subQuery, nextTaskId++);
        scheduledObjectNum++;
        for (Entry<String, List<URI>> eachFetch : fetches.entrySet()) {
          task.addFetches(eachFetch.getKey(), eachFetch.getValue());
          task.addFragment(fragmentsForNonLeafTask[0], true);
          if (fragmentsForNonLeafTask[1] != null) {
            task.addFragment(fragmentsForNonLeafTask[1], true);
          }
        }
        subQuery.getEventHandler().handle(new TaskEvent(task.getId(), TaskEventType.T_SCHEDULE));
      } else if (event instanceof QueryUnitAttemptScheduleEvent) {
        QueryUnitAttemptScheduleEvent castEvent = (QueryUnitAttemptScheduleEvent) event;
        if (context.isLeafQuery()) {
          scheduledRequests.addLeafTask(castEvent);
        } else {

View Full Code Here


    ExecutionBlockId ebId = TajoIdUtils.createExecutionBlockId("eb_000001_00001_00001");


    for (int i = 0; i < 10; i++) {
      QueryUnitId id = new QueryUnitId(ebId, i);
      QueryUnit queryUnit = new QueryUnit(conf, scheduleContext, id, true, null);
      queryUnits.add(queryUnit);


      int launchTime = i + 1;
      int runningTime = i + 1;
      if(i < 9) {
        queryUnit.setLaunchTime(launchTime);
        queryUnit.setFinishTime(launchTime + runningTime);
      }
    }


    Collections.shuffle(queryUnits);

View Full Code Here

TOP

Related Classes of org.apache.tajo.master.querymaster.QueryUnit$InitialScheduleTransition

org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent

org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent

org.apache.tajo.master.DefaultTaskScheduler

org.apache.tajo.master.DefaultTaskScheduler$ScheduledRequests

org.apache.tajo.master.LazyTaskScheduler

org.apache.tajo.master.TaskSchedulerImpl$ScheduledRequests

org.apache.tajo.master.TaskState

org.apache.tajo.storage.DataLocation

org.apache.tajo.util.TestJSPUtil

java.net.URI

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.