Examples of PipelineResult

org.apache.crunch.PipelineResult
Container for the results of a call to {@code run} or {@code done} on thePipeline interface that includes details and statistics about the component stages of the data pipeline.

Examples of org.apache.crunch.PipelineResult

      }
      runCallables(unfinished);
    }
    if (status.get() != Status.FAILED || status.get() != Status.KILLED) {
      status.set(Status.SUCCEEDED);
      set(new PipelineResult(
          ImmutableList.of(new PipelineResult.StageResult("Spark", getCounters(),
              start, System.currentTimeMillis())),
          Status.SUCCEEDED));
    } else {
      set(PipelineResult.EMPTY);

View Full Code Here

Examples of org.apache.crunch.PipelineResult

      public String map(KeyValue input) {
        return input.toString();
      }
    }, strings());
    texts.write(To.textFile(outputPath));
    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    List<String> lines = FileUtils.readLines(new File(outputPath.toString(), "part-m-00000"));
    assertEquals(kvs.size(), lines.size());
    for (int i = 0; i < kvs.size(); i++) {
      assertEquals(kvs.get(i).toString(), lines.get(i));

View Full Code Here

Examples of org.apache.crunch.PipelineResult

    Map<ImmutableBytesWritable, Result> materialized = words.materializeToMap();
    assertEquals(3, materialized.size());
    PCollection<Put> puts = wordCount(words);
    pipeline.write(puts, new HBaseTarget(outputTableName));
    pipeline.write(puts, new HBaseTarget(otherTableName));
    PipelineResult res = pipeline.done();
    assertTrue(res.succeeded());


    assertIsLong(otherTable, "cat", 2);
    assertIsLong(otherTable, "dog", 1);
    assertIsLong(outputTable, "cat", 2);
    assertIsLong(outputTable, "dog", 1);

View Full Code Here

Examples of org.apache.crunch.PipelineResult

    return runtime;
  }


  @Override
  public PipelineResult done() {
    PipelineResult res = super.done();
    if (sparkContext != null) {
      sparkContext.stop();
      sparkContext = null;
    }
    return res;

View Full Code Here

Examples of org.apache.crunch.PipelineResult

        targetDeps.remove(output);
      }
    }
    if (status.get() != Status.FAILED || status.get() != Status.KILLED) {
      status.set(Status.SUCCEEDED);
      set(new PipelineResult(
          ImmutableList.of(new PipelineResult.StageResult("Spark", getCounters(), start, System.currentTimeMillis())),
          Status.SUCCEEDED));
    } else {
      set(PipelineResult.EMPTY);
    }

View Full Code Here

Examples of org.apache.crunch.PipelineResult

    expectedJoinResult.add(Pair.of(222, Pair.of("[Jane Doe,Toilet plunger]", "TOILET PAPER")));
    expectedJoinResult.add(Pair.of(222, Pair.of("[Jane Doe,Toilet plunger]", "TOILET PLUNGER")));
    expectedJoinResult.add(Pair.of(333, Pair.of("[Someone Else,Toilet brush]", "TOILET BRUSH")));
    Iterable<Pair<Integer, Pair<String, String>>> iter = joined.materialize();
    
    PipelineResult res = pipeline.run();
    if (!inMemory) {
      assertEquals(2, res.getStageResults().size());
    }
     
    List<Pair<Integer, Pair<String, String>>> joinedResultList = Lists.newArrayList(iter);
    Collections.sort(joinedResultList);

View Full Code Here

Examples of org.apache.crunch.PipelineResult

    expectedJoinResult.add(Pair.of(222, Pair.of("[Jane Doe,Toilet plunger]", "TOILET PLUNGER")));
    expectedJoinResult.add(Pair.of(333, Pair.of("[Someone Else,Toilet brush]", "TOILET BRUSH")));
    expectedJoinResult.add(Pair.of(444, Pair.<String,String>of("[Has No Orders,null]", null)));
    Iterable<Pair<Integer, Pair<String, String>>> iter = joined.materialize();
    
    PipelineResult res = pipeline.run();
    if (!inMemory) {
      assertEquals(2, res.getStageResults().size());
    }
     
    List<Pair<Integer, Pair<String, String>>> joinedResultList = Lists.newArrayList(iter);
    Collections.sort(joinedResultList);

View Full Code Here

Examples of org.apache.crunch.PipelineResult

        } else if (!failures.isEmpty()) {
          status.set(Status.FAILED);
        } else {
          status.set(Status.SUCCEEDED);
        }
        result = new PipelineResult(stages, status.get());
      }
    } catch (InterruptedException e) {
      throw new AssertionError(e); // Nobody should interrupt us.
    } catch (IOException e) {
      LOG.error("Pipeline failed due to exception", e);

View Full Code Here

Examples of org.apache.crunch.PipelineResult

        return Status.SUCCEEDED;
      }


      @Override
      public PipelineResult getResult() {
        return new PipelineResult(ImmutableList.of(new PipelineResult.StageResult("MemPipelineStage", COUNTERS)),
            Status.SUCCEEDED);
      }


      @Override
      public void kill() {

View Full Code Here

Examples of org.apache.crunch.PipelineResult

  }
  
  @Override
  public PipelineResult run() {
    activeTargets.clear();
    return new PipelineResult(ImmutableList.of(new PipelineResult.StageResult("MemPipelineStage", COUNTERS)),
        PipelineExecution.Status.SUCCEEDED);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.