Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.TezUncheckedException

226
227
228
229
230
231
232
233
      }
      MRHelpers.translateMRConfToTez(conf);
      try {
        return TezUtils.createUserPayloadFromConf(conf);
      } catch (IOException e) {
        throw new TezUncheckedException(e);
      }
    }
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
      uri = new URI(fileName);
    } catch (URISyntaxException e) {
      String message = "Invalid URI defined in configuration for"
          + " location of TEZ jars. providedURI=" + fileName;
      LOG.error(message);
      throw new TezUncheckedException(message, e);
    }

    if (!uri.isAbsolute()) {
      String message = "Non-absolute URI defined in configuration for"
          + " location of TEZ jars. providedURI=" + fileName;
      LOG.error(message);
      throw new TezUncheckedException(message);
    }
    Path p = new Path(uri);
    FileSystem pathfs = p.getFileSystem(conf);
    p = pathfs.makeQualified(p);
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    } else {
      // Add tez jars to local resource
      String[] tezJarUris = conf.getStrings(TezConfiguration.TEZ_LIB_URIS);
       
      if (tezJarUris == null || tezJarUris.length == 0) {
        throw new TezUncheckedException("Invalid configuration of tez jars"
            + ", " + TezConfiguration.TEZ_LIB_URIS
            + " is not defined in the configurartion");
      }
    
      List<Path> tezJarPaths = Lists.newArrayListWithCapacity(tezJarUris.length);

      if (tezJarUris.length == 1 && (tezJarUris[0].endsWith(".tar.gz") || tezJarUris[0].endsWith(".tgz"))) {
        String fileName = tezJarUris[0];
        if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
          FileStatus fStatus = getLRFileStatus(fileName, conf, false)[0];
          LocalResourceVisibility lrVisibility;
          if (checkAncestorPermissionsForAllUsers(conf, fileName, FsAction.EXECUTE) &&
              fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
            lrVisibility = LocalResourceVisibility.PUBLIC;
          } else {
            lrVisibility = LocalResourceVisibility.PRIVATE;
          }
          tezJarResources.put(TezConstants.TEZ_TAR_LR_NAME,
              LocalResource.newInstance(
                  ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                  LocalResourceType.ARCHIVE,
                  lrVisibility,
                  fStatus.getLen(),
                  fStatus.getModificationTime()));
          tezJarPaths.add(fStatus.getPath());
        }
      } else { // Treat as non-archives - each entry being a directory
        for (String tezJarUri : tezJarUris) {
          boolean ancestorsHavePermission = checkAncestorPermissionsForAllUsers(conf, tezJarUri,
              FsAction.EXECUTE);
          FileStatus [] fileStatuses = getLRFileStatus(tezJarUri, conf, true);
          for (FileStatus fStatus : fileStatuses) {
            if (fStatus.isDirectory()) {
              // Skip directories - since tez.lib.uris is not recursive.
              continue;
            }
            LocalResourceVisibility lrVisibility;
            if (ancestorsHavePermission &&
                fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
              lrVisibility = LocalResourceVisibility.PUBLIC;
            } else {
              lrVisibility = LocalResourceVisibility.PRIVATE;
            }
            String rsrcName = fStatus.getPath().getName();
            // FIXME currently not checking for duplicates due to quirks
            // in assembly generation
            if (tezJarResources.containsKey(rsrcName)) {
              String message = "Duplicate resource found"
                  + ", resourceName=" + rsrcName
                  + ", existingPath=" +
                  tezJarResources.get(rsrcName).getResource().toString()
                  + ", newPath=" + fStatus.getPath();
              LOG.warn(message);
              // throw new TezUncheckedException(message);
            }
            tezJarResources.put(rsrcName,
                LocalResource.newInstance(
                    ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                    LocalResourceType.FILE,
                    lrVisibility,
                    fStatus.getLen(),
                    fStatus.getModificationTime()));
          }
        }
      }
     
      if (tezJarResources.isEmpty()) {
        throw new TezUncheckedException(
            "No files found in locations specified in "
                + TezConfiguration.TEZ_LIB_URIS + " . Locations: "
                + StringUtils.join(tezJarUris, ','));
      } else {
        // Obtain credentials.
View Full Code Here
783
784
785
786
787
788
789
790
791
792
793
    try {
      appReport = yarnClient.getApplicationReport(
          applicationId);

      if(appReport == null) {
        throw new TezUncheckedException("Could not retrieve application report"
            + " from YARN, applicationId=" + applicationId);
      }
      YarnApplicationState appState = appReport.getYarnApplicationState();
      if(appState != YarnApplicationState.RUNNING) {
        if (appState == YarnApplicationState.FINISHED
View Full Code Here
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
        break;
      case SUCCEEDED:
        dagState = DAGStatusStateProto.DAG_SUCCEEDED;
        break;
      default:
        throw new TezUncheckedException("Encountered unknown final application"
          + " status from YARN"
          + ", appState=" + appReport.getYarnApplicationState()
          + ", finalStatus=" + appReport.getFinalApplicationStatus());
      }
      break;
    default:
      throw new TezUncheckedException("Encountered unknown application state"
          + " from YARN, appState=" + appReport.getYarnApplicationState());
    }

    builder.setState(dagState);
    if(appReport.getDiagnostics() != null) {
View Full Code Here
83
84
85
86
87
88
89
90
91
        mkDirForAM(fs, p);
        LOG.info("Stage directory " + p + " doesn't exist and is created");
      }
      baseStagingDir = fs.resolvePath(p);
    } catch (IOException e) {
      throw new TezUncheckedException(e);
    }
    return baseStagingDir;
  }
View Full Code Here
115
116
117
118
119
120
121
122
123
      if (!fs.exists(tezStagingDir)) {
        mkDirForAM(fs, tezStagingDir);
        LOG.info("Tez system stage directory " + tezStagingDir + " doesn't exist and is created");
      }
    } catch (IOException e) {
      throw new TezUncheckedException(e);
    }
    return tezStagingDir;
  }
View Full Code Here
302
303
304
305
306
307
308
309
310
311
312
  public static void addAdditionalLocalResources(Map<String, LocalResource> additionalLrs,
      Map<String, LocalResource> originalLRs) {
    if (additionalLrs != null && !additionalLrs.isEmpty()) {
      for (Map.Entry<String, LocalResource> lr : additionalLrs.entrySet()) {
        if (originalLRs.containsKey(lr.getKey())) {
          throw new TezUncheckedException("Attempting to add duplicate resource: " + lr.getKey());
        } else {
          originalLRs.put(lr.getKey(), lr.getValue());
        }
      }
    }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
    Class<?> clazz = CLAZZ_CACHE.get(className);
    if (clazz == null) {
      try {
        clazz = Class.forName(className, true, Thread.currentThread().getContextClassLoader());
      } catch (ClassNotFoundException e) {
        throw new TezUncheckedException("Unable to load class: " + className, e);
      }
    }
    return clazz;
  }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
  private static <T> T getNewInstance(Class<T> clazz) {
    T instance;
    try {
      instance = clazz.newInstance();
    } catch (InstantiationException e) {
      throw new TezUncheckedException(
          "Unable to instantiate class with 0 arguments: " + clazz.getName(), e);
    } catch (IllegalAccessException e) {
      throw new TezUncheckedException(
          "Unable to instantiate class with 0 arguments: " + clazz.getName(), e);
    }
    return instance;
  }
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.TezUncheckedException

  • org.apache.hadoop.mapred.MRVertexOutputCommitter
  • org.apache.hadoop.mapred.split.TezGroupedSplit
  • org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat
  • org.apache.hadoop.mapred.split.TezMapredSplitsGrouper
  • org.apache.hadoop.mapreduce.split.TezGroupedSplit
  • org.apache.hadoop.mapreduce.split.TezGroupedSplitsInputFormat
  • org.apache.hadoop.mapreduce.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader
  • org.apache.hadoop.mapreduce.split.TezMapReduceSplitsGrouper
  • org.apache.tez.client.AMConfiguration
  • org.apache.tez.client.TezClientUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.