Examples of cascading.tap.hadoop.Hfs$CombinedInputFormat

cascading.tap.hadoop.Hfs
Combined input format that uses the underlying individual input format to combine multiple files into a single split.

    }


  @Test
  public void testNestedProperties() throws IOException
    {
    Tap source = new Hfs( new TextLine( new Fields( "line" ) ), "/input" );


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third" ), "\\s" ), Fields.ALL );


    Tap sink = new Hfs( new TextLine(), "output", true );


    Properties defaultProperties = new Properties();


    defaultProperties.setProperty( "test.key", "test.value" );

View Full Code Here

    }


  @Test
  public void testEveryAfterJoin()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge1" );
    Tap source2 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge2" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe left = new Each( new Pipe( "left" ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );


    Fields[] fields = Fields.fields( new Fields( "offset" ), new Fields( "offset" ) );

View Full Code Here

    for( int i = 0; i < n; i++ )
      {
      String nameIn = "in" + i;
      String nameOut = "out" + i;
      Pipe pipe = new Pipe( nameIn );
      sources.put( nameIn, new Hfs( new TextLine( new Fields( "key" + i ) ), "foo/in" + i ) );
      sinks.put( nameOut, new Hfs( new TextLine(), "foo/out" + i ) );


      count += 2; // 2 taps


      if( i > 0 )
        {

View Full Code Here


  @Test
  public void testLocalModeSource() throws Exception
    {
    Tap source = new Lfs( new TextLine(), "input/path" );
    Tap sink = new Hfs( new TextLine(), "output/path", SinkMode.REPLACE );


    Pipe pipe = new Pipe( "test" );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );

View Full Code Here

  // test is not executed, just guarantees flow is run locally


  @Test
  public void testLocalModeSink() throws Exception
    {
    Tap source = new Hfs( new TextLine(), "input/path" );
    Tap sink = new Lfs( new TextLine(), "output/path", SinkMode.REPLACE );


    Pipe pipe = new Pipe( "test" );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );

View Full Code Here

  public void testNotLocalMode() throws Exception
    {
    if( !getPlatform().isUseCluster() )
      return;


    Tap source = new Hfs( new TextLine(), "input/path" );
    Tap sink = new Hfs( new TextLine(), "output/path", true );


    Pipe pipe = new Pipe( "test" );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );

View Full Code Here

      return;


    getPlatform().copyFromLocal( inputFileLower );
    getPlatform().copyFromLocal( inputFileUpper );


    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower );
    Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileUpper );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );


    Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), getOutputPath( "stopped" ), true );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );


    pipeLower = new GroupBy( pipeLower, new Fields( "num" ) );

View Full Code Here

  @Test
  public void testFailedSerialization() throws Exception
    {
    getPlatform().copyFromLocal( inputFileLower );


    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );


    Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), getOutputPath( "badserialization" ), true );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );


    pipeLower = new Each( pipeLower, new Fields( "num" ), new BadFilter() );

View Full Code Here

  @Test
  public void testStartStopRace() throws Exception
    {
    getPlatform().copyFromLocal( inputFileLower );


    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );


    Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), getOutputPath( "startstop" ), SinkMode.REPLACE );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );


    pipeLower = new GroupBy( pipeLower, new Fields( "num" ) );

View Full Code Here

  private void failingListenerTest( FailingFlowListener.OnFail onFail ) throws Exception
    {
    getPlatform().copyFromLocal( inputFileLower );
    getPlatform().copyFromLocal( inputFileUpper );


    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower );
    Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileUpper );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );


    Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), getOutputPath( onFail + "/stopped" ), true );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );


    if( onFail == FailingFlowListener.OnFail.THROWABLE )
      {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.tap.hadoop.Hfs$CombinedInputFormat

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.hadoop.FlowPlatformTest

cascading.flow.hadoop.MapReduceFlow

cascading.flow.hadoop.MapReduceFlowPlatformTest

cascading.flow.hadoop.planner.rule.scopeexpression.EquivalentTapsScopeExpression

cascading.flow.hadoop.util.HadoopMRUtil

cascading.flow.hadoop.util.HadoopUtil

cascading.lingual.platform.hadoop.HadoopDefaultFactory

cascading.lingual.platform.hadoop.HadoopPlatformBroker

cascading.lingual.platform.hadoop2.Hadoop2MR1DefaultFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.