Examples of org.apache.drill.exec.planner.fragment.SimpleParallelizer

Package org.apache.drill.exec.planner.fragment

Examples of org.apache.drill.exec.planner.fragment.SimpleParallelizer

org.apache.drill.exec.planner.fragment.SimpleParallelizer
The simple parallelizer determines the level of parallelization of a plan based on the cost of the underlying operations. It doesn't take into account system load or other factors. Based on the cost of the query, the parallelization for each major fragment will be determined. Once the amount of parallelization is done, assignment is done based on round robin assignment ordered by operator affinity (locality) to available execution Drillbits.

    
    


    
    PlanningSet planningSet = StatsCollector.collectStats(rootFragment);
    SimpleParallelizer parallelizer = new SimpleParallelizer();


    try {
      QueryWorkUnit work = parallelizer.getFragments(context.getCurrentEndpoint(), queryId, context.getActiveEndpoints(), context.getPlanReader(), rootFragment, planningSet, 10);


      this.context.getBitCom().getListeners().addFragmentStatusListener(work.getRootFragment().getHandle(), fragmentManager);
      List<PlanFragment> leafFragments = Lists.newArrayList();
      List<PlanFragment> intermediateFragments = Lists.newArrayList();

View Full Code Here

    
    System.out.println(String.format("=================Building plan fragments for [%s].  Allowing %d total Drillbits.==================", fragmentFile, bitCount));
    PhysicalPlanReader ppr = new PhysicalPlanReader(CONFIG, CONFIG.getMapper(), DrillbitEndpoint.getDefaultInstance());
    Fragment fragmentRoot = getRootFragment(ppr, fragmentFile);
    PlanningSet planningSet = StatsCollector.collectStats(fragmentRoot);
    SimpleParallelizer par = new SimpleParallelizer();
    List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    DrillbitEndpoint localBit = null;
    for(int i =0; i < bitCount; i++){
      DrillbitEndpoint b1 = DrillbitEndpoint.newBuilder().setAddress("localhost").setBitPort(1234+i).build();
      if(i ==0) localBit = b1; 
      endpoints.add(b1);
    }
    
    
    QueryWorkUnit qwu = par.getFragments(localBit, QueryId.getDefaultInstance(), endpoints, ppr, fragmentRoot, planningSet, 10);
    System.out.println(String.format("=========ROOT FRAGMENT [%d:%d] =========", qwu.getRootFragment().getHandle().getMajorFragmentId(), qwu.getRootFragment().getHandle().getMinorFragmentId()));
    
    System.out.print(qwu.getRootFragment().getFragmentJson());

View Full Code Here


    System.out.println(String.format("=================Building plan fragments for [%s].  Allowing %d total Drillbits.==================", fragmentFile, bitCount));
    PhysicalPlanReader ppr = new PhysicalPlanReader(CONFIG, CONFIG.getMapper(), DrillbitEndpoint.getDefaultInstance());
    Fragment fragmentRoot = getRootFragment(ppr, fragmentFile);
    PlanningSet planningSet = StatsCollector.collectStats(fragmentRoot);
    SimpleParallelizer par = new SimpleParallelizer(1000*1000, 5, 10, 1.2);
    List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    DrillbitEndpoint localBit = null;
    for(int i =0; i < bitCount; i++){
      DrillbitEndpoint b1 = DrillbitEndpoint.newBuilder().setAddress("localhost").setControlPort(1234+i).build();
      if(i ==0) localBit = b1;
      endpoints.add(b1);
    }


    QueryWorkUnit qwu = par.getFragments(new OptionList(), localBit, QueryId.getDefaultInstance(), endpoints, ppr, fragmentRoot, planningSet);
    System.out.println(String.format("=========ROOT FRAGMENT [%d:%d] =========", qwu.getRootFragment().getHandle().getMajorFragmentId(), qwu.getRootFragment().getHandle().getMinorFragmentId()));


    System.out.print(qwu.getRootFragment().getFragmentJson());

View Full Code Here

      fail("Failure while fragmenting query.", e);
      return;
    }


    PlanningSet planningSet = StatsCollector.collectStats(rootFragment);
    SimpleParallelizer parallelizer = new SimpleParallelizer(context);


    try {


      double size = 0;
      for(PhysicalOperator ops : plan.getSortedOperators()){
        size += ops.getCost();
      }
      if(queuingEnabled){
        if(size > this.queueThreshold){
          this.lease = largeSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
        }else{
          this.lease = smallSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
        }
      }


      QueryWorkUnit work = parallelizer.getFragments(context.getOptions().getSessionOptionList(), context.getCurrentEndpoint(),
          queryId, context.getActiveEndpoints(), context.getPlanReader(), rootFragment, planningSet);


      this.context.getWorkBus().setFragmentStatusListener(work.getRootFragment().getHandle().getQueryId(), fragmentManager);
      List<PlanFragment> leafFragments = Lists.newArrayList();
      List<PlanFragment> intermediateFragments = Lists.newArrayList();

View Full Code Here

        }
      }
    }


    PlanningSet planningSet = StatsCollector.collectStats(rootFragment);
    SimpleParallelizer parallelizer = new SimpleParallelizer(context);


    try {


      double size = 0;
      for (PhysicalOperator ops : plan.getSortedOperators()) {
        size += ops.getCost();
      }
      if (queuingEnabled) {
        if (size > this.queueThreshold) {
          this.lease = largeSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
        } else {
          this.lease = smallSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
        }
      }


      QueryWorkUnit work = parallelizer.getFragments(context.getOptions().getOptionList(), context.getCurrentEndpoint(),
          queryId, context.getActiveEndpoints(), context.getPlanReader(), rootFragment, planningSet, initiatingClient.getSession());


      this.context.getWorkBus().setFragmentStatusListener(work.getRootFragment().getHandle().getQueryId(), fragmentManager);


      int totalFragments = 1 + work.getFragments().size();;

View Full Code Here


    System.out.println(String.format("=================Building plan fragments for [%s].  Allowing %d total Drillbits.==================", fragmentFile, bitCount));
    PhysicalPlanReader ppr = new PhysicalPlanReader(CONFIG, CONFIG.getMapper(), DrillbitEndpoint.getDefaultInstance());
    Fragment fragmentRoot = getRootFragment(ppr, fragmentFile);
    PlanningSet planningSet = StatsCollector.collectStats(fragmentRoot);
    SimpleParallelizer par = new SimpleParallelizer(1000*1000, 5, 10, 1.2);
    List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    DrillbitEndpoint localBit = null;
    for(int i =0; i < bitCount; i++) {
      DrillbitEndpoint b1 = DrillbitEndpoint.newBuilder().setAddress("localhost").setControlPort(1234+i).build();
      if (i == 0) {
        localBit = b1;
      }
      endpoints.add(b1);
    }


    QueryWorkUnit qwu = par.getFragments(new OptionList(), localBit, QueryId.getDefaultInstance(), endpoints, ppr, fragmentRoot, planningSet,
        UserSession.Builder.newBuilder().withCredentials(UserBitShared.UserCredentials.newBuilder().setUserName("foo").build()).build());
    System.out.println(String.format("=========ROOT FRAGMENT [%d:%d] =========", qwu.getRootFragment().getHandle().getMajorFragmentId(), qwu.getRootFragment().getHandle().getMinorFragmentId()));


    System.out.print(qwu.getRootFragment().getFragmentJson());

View Full Code Here

        }
      }
    }


    PlanningSet planningSet = StatsCollector.collectStats(rootFragment);
    SimpleParallelizer parallelizer = new SimpleParallelizer(context);


    try {


      double size = 0;
      for(PhysicalOperator ops : plan.getSortedOperators()){
        size += ops.getCost();
      }
      if(queuingEnabled){
        if(size > this.queueThreshold){
          this.lease = largeSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
        }else{
          this.lease = smallSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
        }
      }


      QueryWorkUnit work = parallelizer.getFragments(context.getOptions().getOptionList(), context.getCurrentEndpoint(),
          queryId, context.getActiveEndpoints(), context.getPlanReader(), rootFragment, planningSet);


      this.context.getWorkBus().setFragmentStatusListener(work.getRootFragment().getHandle().getQueryId(), fragmentManager);
      List<PlanFragment> leafFragments = Lists.newArrayList();
      List<PlanFragment> intermediateFragments = Lists.newArrayList();

View Full Code Here

TOP

Related Classes of org.apache.drill.exec.planner.fragment.SimpleParallelizer

org.apache.drill.exec.physical.base.FragmentRoot

org.apache.drill.exec.physical.base.PhysicalOperator

org.apache.drill.exec.planner.fragment.Materializer.IndexedFragmentNode

org.apache.drill.exec.pop.TestFragmentChecker

org.apache.drill.exec.proto.BitControl.PlanFragment

org.apache.drill.exec.proto.ExecProtos.FragmentHandle

org.apache.drill.exec.proto.ExecProtos.PlanFragment

org.apache.drill.exec.work.foreman.Foreman

org.apache.drill.exec.work.QueryWorkUnit

org.apache.drill.exec.exception.FragmentSetupException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.