Package org.apache.crunch

Examples of org.apache.crunch.CrunchRuntimeException

84
85
86
87
88
89
90
91
        // Hacking this for Hadoop1 and Hadoop2
        getConfiguration().set("mapreduce.job.cache.local.files", sparkFiles);
        getConfiguration().set("mapred.cache.localFiles", sparkFiles);
      }
    } catch (IOException e) {
      throw new CrunchRuntimeException(e);
    }
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
      try {
        Job job = new Job(ctxt.getConfiguration());
        ptype.configureShuffle(job, options);
        cmp = ReflectionUtils.newInstance(options.getGroupingComparatorClass(), job.getConfiguration());
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error configuring grouping comparator", e);
      }
    }
    return cmp;
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
          if (cmp == null) {
            cmp = new ByteWritable.Comparator();
          }
        }
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error configuring sort comparator", e);
      }
    }
    return cmp;
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
        ptype.initialize(runtimeContext.getConfiguration());
        Job job = new Job(runtimeContext.getConfiguration());
        ptype.configureShuffle(job, GroupingOptions.builder().partitionerClass(partitionerClass).build());
        partitioner = ReflectionUtils.newInstance(partitionerClass, job.getConfiguration());
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error configuring partitioner", e);
      }
    }
    return partitioner;
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
   
    try {
      TableMapReduceUtil.addDependencyJars(job);
      FileOutputFormat.setOutputPath(job, outputPath);
    } catch (IOException e) {
      throw new CrunchRuntimeException(e);
    }

    if (null == name) {
      job.setOutputFormatClass(TableOutputFormat.class);
      job.setOutputKeyClass(ImmutableBytesWritable.class);
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
    }
    if (materializedAt != null && (materializedAt instanceof ReadableSource)) {
      try {
        return ((ReadableSource<S>) materializedAt).read(getPipeline().getConfiguration());
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error reading materialized data", e);
      }
    }
    materialized = true;
    return pipeline.materialize(this);
  }
View Full Code Here
231
232
233
234
235
236
237
238
239
      }
      System.arraycopy(bb.array(), bb.position(), bytes, 0, len);
      try {
        deserializer.deserialize(next, bytes);
      } catch (TException e) {
        throw new CrunchRuntimeException(e);
      }
      return next;
    }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
      try {
        Class<?> keyClass = Class.forName(fields.get(2));
        Class<?> valueClass = Class.forName(fields.get(3));
        out.put(name, new OutputConfig(bundle, keyClass, valueClass));
      } catch (ClassNotFoundException e) {
        throw new CrunchRuntimeException(e);
      }
    }
    return out;
  }
View Full Code Here
254
255
256
257
258
259
260
261
    @Override
    public ByteBuffer map(T t) {
      try {
        return ByteBuffer.wrap(serializer.serialize(t));
      } catch (TException e) {
        throw new CrunchRuntimeException(e);
      }
    }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
      Path targetPath = new Path(target.toString());
      ObjectInputStream ois = new ObjectInputStream(targetPath.getFileSystem(conf).open(targetPath));
      try {
        value = ois.readObject();
      } catch (ClassNotFoundException e) {
        throw new CrunchRuntimeException(e);
      }
      ois.close();
    }
    return value;
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.CrunchRuntimeException

  • com.twitter.elephantbird.crunch.CrunchElephantBirdExample$PersonInAddressBookFn
  • org.apache.crunch.fn.SDoubleFlatMapFunction
  • org.apache.crunch.fn.SDoubleFunction
  • org.apache.crunch.fn.SFlatMapFunction
  • org.apache.crunch.fn.SFlatMapFunction2
  • org.apache.crunch.fn.SFunction
  • org.apache.crunch.fn.SFunction2
  • org.apache.crunch.fn.SPairFlatMapFunction
  • org.apache.crunch.fn.SPairFunction
  • org.apache.crunch.impl.dist.collect.PCollectionImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.