Examples of OutputContext

net.kuujo.vertigo.io.OutputContext
com/kuujo">Jordan Halterman
org.apache.tez.runtime.api.OutputContext
Context handle for the Output to initialize itself. This interface is not supposed to be implemented by users
org.jnode.install.OutputContext
@author Levente S\u00e1ntha
org.springframework.data.hadoop.store.support.OutputContext
Context strategy keeping output state. @author Janne Valkealahti

Examples of org.apache.tez.runtime.api.OutputContext




    OutputDescriptor outputDescriptor = mock(OutputDescriptor.class);
    when(outputDescriptor.getClassName()).thenReturn("OutputDescriptor");


    OutputContext outputContext = new TezOutputContextImpl(conf, new String[] {workDir.toString()},
        appAttemptNumber, tezUmbilical, dagName, taskVertexName, destinationVertexName,
        -1, taskAttemptID, counters, 0, userPayload, runtimeTask,
        null, auxEnv, new MemoryDistributor(1, 1, conf) , outputDescriptor, null);


    UnorderedKVOutput kvOutput = new OnFileUnorderedKVOutputForTest(outputContext, 1);


    List<Event> events = null;


    events = kvOutput.initialize();
    assertTrue(events != null && events.size() == 0);


    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }


    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);


    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());


    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));


    assertFalse(shufflePayload.hasEmptyPartitions());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.OutputContext

    InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);


    // Third request - randomOutput (simulates MROutput)
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
    dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);


    // Fourth request - OnFileSortedOutput
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
    OutputContext e4OutputContext2 = createTestOutputContext();
    OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
    dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);


    dist.makeInitialAllocations();

View Full Code Here

Examples of org.apache.tez.runtime.api.OutputContext

    InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);


    // Third request - randomOutput (simulates MROutput) [weight 1]
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
    dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);


    // Fourth request - OnFileSortedOutput [weight 3]
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
    OutputContext e4OutputContext2 = createTestOutputContext();
    OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
    dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);


    dist.makeInitialAllocations();

View Full Code Here

Examples of org.apache.tez.runtime.api.OutputContext


    return parameters;
  }


  private void startSortedOutput(int partitions) throws Exception {
    OutputContext context = createTezOutputContext();
    sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
    sortedOutput.initialize();
    sortedOutput.start();
    writer = sortedOutput.getWriter();
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.OutputContext

    DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer();
    serviceProviderMetaData.writeInt(PORT);


    TezCounters counters = new TezCounters();


    OutputContext context = mock(OutputContext.class);
    doReturn(counters).when(context).getCounters();
    doReturn(workingDirs).when(context).getWorkDirs();
    doReturn(payLoad).when(context).getUserPayload();
    doReturn(5 * 1024 * 1024l).when(context).getTotalMemoryAvailableToTask();
    doReturn(UniqueID).when(context).getUniqueIdentifier();

View Full Code Here

Examples of org.apache.tez.runtime.api.OutputContext

        inputSpec.getInputDescriptor(), inputMap, inputReadyTracker, objectRegistry);
    return inputContext;
  }


  private OutputContext createOutputContext(OutputSpec outputSpec, int outputIndex) {
    OutputContext outputContext = new TezOutputContextImpl(tezConf, localDirs,
        appAttemptNumber, tezUmbilical,
        taskSpec.getDAGName(), taskSpec.getVertexName(),
        outputSpec.getDestinationVertexName(),
        taskSpec.getVertexParallelism(),
        taskSpec.getTaskAttemptID(),

View Full Code Here

Examples of org.apache.tez.runtime.api.OutputContext


    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Output using OutputSpec: " + outputSpec);
      String edgeName = outputSpec.getDestinationVertexName();
      OutputContext outputContext = createOutputContext(outputSpec, outputIndex);
      LogicalOutput output = createOutput(outputSpec, outputContext);


      outputsMap.put(edgeName, output);
      outputContextMap.put(edgeName, outputContext);


      LOG.info("Initializing Output with dest edge: " + edgeName);
      List<Event> events = ((OutputFrameworkInterface)output).initialize();
      sendTaskGeneratedEvents(events, EventProducerConsumerType.OUTPUT,
          outputContext.getTaskVertexName(),
          outputContext.getDestinationVertexName(), taskSpec.getTaskAttemptID());
      LOG.info("Initialized Output with dest edge: " + edgeName);
      return null;
    }

View Full Code Here

Examples of org.jnode.install.OutputContext

            }
        };
    }


    protected OutputContext getOutputContext() {
        return new OutputContext() {
            public void showMessage(String msg) {
                System.out.println(msg);
            }
        };
    }

View Full Code Here

Examples of org.springframework.data.hadoop.store.support.OutputContext

    OutputStream out = streamsHolder.getStream();
    out.write(entity.getBytes());
    out.write(delimiter);
    setWritePosition(getPosition(streamsHolder));


    OutputContext context = getOutputContext();
    if (context.getRolloverState()) {
      log.info("after write, rollever state is true");
      close();
      context.rollStrategies();
    }
  }

View Full Code Here

Examples of org.springframework.data.hadoop.store.support.OutputContext

    OutputStream out = streamsHolder.getStream();
    out.write(entity);


    setWritePosition(getPosition(streamsHolder));


    OutputContext context = getOutputContext();
    if (context.getRolloverState()) {
      log.info("after write, rollever state is true");
      close();
      context.rollStrategies();
    }


  }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.