Package org.apache.crunch

Examples of org.apache.crunch.ParallelDoOptions

120
121
122
123
124
125
126
127
128
129
130
  private PTable<K, Pair<U,V>> joinInternal(PTable<K, U> left, PTable<K, V> right, boolean includeUnmatchedLeftValues) {
    PTypeFamily tf = left.getTypeFamily();
    ReadableData<Pair<K, V>> rightReadable = right.asReadable(materialize);
    MapsideJoinDoFn<K, U, V> mapJoinDoFn = new MapsideJoinDoFn<K, U, V>(
              rightReadable, right.getPTableType(), includeUnmatchedLeftValues);
    ParallelDoOptions options = ParallelDoOptions.builder()
        .sourceTargets(rightReadable.getSourceTargets())
        .build();
    return left.parallelDo("mapjoin", mapJoinDoFn,
        tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
        options);
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
  @Test
  public void testFormatNodePath() {
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);
    ParallelDoOptions doOptions = ParallelDoOptions.builder().build();

    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(doOptions);
    when(head.getParallelDoOptions()).thenReturn(doOptions);
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    SourceTarget<?> srcTarget = mock(SourceTarget.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);

    ParallelDoOptions tailOptions = ParallelDoOptions.builder().sourceTargets(srcTarget).build();
    ParallelDoOptions headOptions = ParallelDoOptions.builder().build();
    when(srcTarget.toString()).thenReturn("target");
    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(tailOptions);
    when(head.getParallelDoOptions()).thenReturn(headOptions);
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
  @Test
  public void testFormatNodePath() {
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);
    ParallelDoOptions doOptions = ParallelDoOptions.builder().build();

    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(doOptions);
    when(head.getParallelDoOptions()).thenReturn(doOptions);
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    SourceTarget<?> srcTarget = mock(SourceTarget.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);

    ParallelDoOptions tailOptions = ParallelDoOptions.builder().sourceTargets(srcTarget).build();
    ParallelDoOptions headOptions = ParallelDoOptions.builder().build();
    when(srcTarget.toString()).thenReturn("target");
    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(tailOptions);
    when(head.getParallelDoOptions()).thenReturn(headOptions);
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
  private PTable<K, Pair<U,V>> joinInternal(PTable<K, U> left, PTable<K, V> right, boolean includeUnmatchedLeftValues) {
    PTypeFamily tf = left.getTypeFamily();
    ReadableData<Pair<K, V>> rightReadable = right.asReadable(materialize);
    MapsideJoinDoFn<K, U, V> mapJoinDoFn = new MapsideJoinDoFn<K, U, V>(
              rightReadable, right.getPTableType(), includeUnmatchedLeftValues);
    ParallelDoOptions options = ParallelDoOptions.builder()
        .sourceTargets(rightReadable.getSourceTargets())
        .build();
    return left.parallelDo("mapjoin", mapJoinDoFn,
        tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
        options);
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
  private PTable<K, Pair<U,V>> joinInternal(PTable<K, U> left, PTable<K, V> right, boolean includeUnmatchedLeftValues) {
    PTypeFamily tf = left.getTypeFamily();
    ReadableData<Pair<K, V>> rightReadable = right.asReadable(materialize);
    MapsideJoinDoFn<K, U, V> mapJoinDoFn = new MapsideJoinDoFn<K, U, V>(rightReadable, includeUnmatchedLeftValues);
    ParallelDoOptions options = ParallelDoOptions.builder()
        .sourceTargets(rightReadable.getSourceTargets())
        .build();
    return left.parallelDo("mapjoin", mapJoinDoFn,
        tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
        options);
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
  @Test
  public void testFormatNodePath() {
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);
    ParallelDoOptions doOptions = ParallelDoOptions.builder().build();

    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(doOptions);
    when(head.getParallelDoOptions()).thenReturn(doOptions);
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
    PCollectionImpl<?> tail = mock(PCollectionImpl.class);
    PCollectionImpl<?> head = mock(PCollectionImpl.class);
    SourceTarget<?> srcTarget = mock(SourceTarget.class);
    JobPrototype jobPrototype = mock(JobPrototype.class);

    ParallelDoOptions tailOptions = ParallelDoOptions.builder().sourceTargets(srcTarget).build();
    ParallelDoOptions headOptions = ParallelDoOptions.builder().build();
    when(srcTarget.toString()).thenReturn("target");
    when(tail.getName()).thenReturn("tail");
    when(head.getName()).thenReturn("head");
    when(tail.getParallelDoOptions()).thenReturn(tailOptions);
    when(head.getParallelDoOptions()).thenReturn(headOptions);
View Full Code Here
TOP

Related Classes of org.apache.crunch.ParallelDoOptions

  • org.apache.crunch.impl.mr.plan.DotfileWriterTest
  • org.apache.crunch.lib.join.MapsideJoinStrategy
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.