Package org.apache.hadoop.hive.ql.plan

Examples of org.apache.hadoop.hive.ql.plan.MapredLocalWork


      assert plan.getAliasToWork().get(alias_id) == null;
      plan.getAliasToWork().put(alias_id, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
            new LinkedHashMap<String, Operator<? extends Serializable>>(),
            new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias_id) == null;
      assert localPlan.getAliasToFetchWork().get(alias_id) == null;
      localPlan.getAliasToWork().put(alias_id, topOp);
      if (tblDir == null)
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(fetchWork.convertPathToStringArray(partDir), partDesc));
      else
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(tblDir.toString(), tblDesc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here


      plan.getPathToPartitionInfo().put(path, new partitionDesc(tt_desc, null));
      plan.getAliasToWork().put(alias, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
                                        new LinkedHashMap<String, Operator<? extends Serializable>>(),
                                        new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias) == null;
      assert localPlan.getAliasToFetchWork().get(alias) == null;
      localPlan.getAliasToWork().put(alias, topOp);
      localPlan.getAliasToFetchWork().put(alias, new fetchWork(alias, tt_desc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here

      mo.setChildren(job);
      l4j.info(mo.dump(0));
      mo.initialize(jc, null);

      // initialize map local work
      mapredLocalWork localWork = mrwork.getMapLocalWork();
      if (localWork == null) {
        return;
      }
      fetchOperators = new HashMap<String, FetchOperator>();
      // create map local operators
      for (Map.Entry<String, fetchWork> entry : localWork.getAliasToFetchWork().entrySet()) {
        fetchOperators.put(entry.getKey(), new FetchOperator(entry.getValue(), job));
        l4j.info("fetchoperator for " + entry.getKey() + " created");
      }
      // initialize map local operators
      for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
        Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
        // All the operators need to be initialized before process
        forwardOp.initialize(jc, new ObjectInspector[]{entry.getValue().getOutputObjectInspector()});
        l4j.info("fetchoperator for " + entry.getKey() + " initialized");
      }
      // defer processing of map local operators to first row if in case there is no input (??)
View Full Code Here

      mo.setOutputCollector(oc);
      mo.setReporter(rp);
      // process map local operators
      if (fetchOperators != null) {
        try {
          mapredLocalWork localWork = mo.getConf().getMapLocalWork();
          int fetchOpNum = 0;
          for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
            int fetchOpRows = 0;
            String alias = entry.getKey();
            FetchOperator fetchOp = entry.getValue();
            Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);

            while (true) {
              InspectableObject row = fetchOp.getNextRow();
              if (row == null) {
                forwardOp.close(false);
View Full Code Here

    // detecting failed executions by exceptions thrown by the operator tree
    // ideally hadoop should let us know whether map execution failed or not
    try {
      mo.close(abort);
      if (fetchOperators != null) {
        mapredLocalWork localWork = mo.getConf().getMapLocalWork();
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
          forwardOp.close(abort);
        }
      }
     
      if (l4j.isInfoEnabled()) {
View Full Code Here

      mo.setChildren(job);
      l4j.info(mo.dump(0));
      mo.initialize(jc, null);

      // initialize map local work
      mapredLocalWork localWork = mrwork.getMapLocalWork();
      if (localWork == null) {
        return;
      }
      fetchOperators = new HashMap<String, FetchOperator>();
      // create map local operators
      for (Map.Entry<String, fetchWork> entry : localWork.getAliasToFetchWork().entrySet()) {
        fetchOperators.put(entry.getKey(), new FetchOperator(entry.getValue(), job));
        l4j.info("fetchoperator for " + entry.getKey() + " created");
      }
      // initialize map local operators
      for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
        Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
        // All the operators need to be initialized before process
        forwardOp.initialize(jc, new ObjectInspector[]{entry.getValue().getOutputObjectInspector()});
        l4j.info("fetchoperator for " + entry.getKey() + " initialized");
      }
      // defer processing of map local operators to first row if in case there is no input (??)
View Full Code Here

      mo.setOutputCollector(oc);
      mo.setReporter(rp);
      // process map local operators
      if (fetchOperators != null) {
        try {
          mapredLocalWork localWork = mo.getConf().getMapLocalWork();
          int fetchOpNum = 0;
          for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
            int fetchOpRows = 0;
            String alias = entry.getKey();
            FetchOperator fetchOp = entry.getValue();
            Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);

            while (true) {
              InspectableObject row = fetchOp.getNextRow();
              if (row == null) {
                break;
View Full Code Here

    // detecting failed executions by exceptions thrown by the operator tree
    // ideally hadoop should let us know whether map execution failed or not
    try {
      mo.close(abort);
      if (fetchOperators != null) {
        mapredLocalWork localWork = mo.getConf().getMapLocalWork();
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
          forwardOp.close(abort);
        }
      }
     
      if (l4j.isInfoEnabled()) {
View Full Code Here

      assert plan.getAliasToWork().get(alias_id) == null;
      plan.getAliasToWork().put(alias_id, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
            new LinkedHashMap<String, Operator<? extends Serializable>>(),
            new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias_id) == null;
      assert localPlan.getAliasToFetchWork().get(alias_id) == null;
      localPlan.getAliasToWork().put(alias_id, topOp);
      if (tblDir == null)
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(fetchWork.convertPathToStringArray(partDir), partDesc));
      else
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(tblDir.toString(), tblDesc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here

      plan.getPathToPartitionInfo().put(path, new partitionDesc(tt_desc, null));
      plan.getAliasToWork().put(alias, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
                                        new LinkedHashMap<String, Operator<? extends Serializable>>(),
                                        new LinkedHashMap<String, fetchWork>());
     
      assert localPlan.getAliasToWork().get(alias) == null;
      assert localPlan.getAliasToFetchWork().get(alias) == null;
      localPlan.getAliasToWork().put(alias, topOp);
      localPlan.getAliasToFetchWork().put(alias, new fetchWork(alias, tt_desc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.ql.plan.MapredLocalWork

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.