Examples of cascading.tap.hadoop.Hfs$CombinedInputFormat

cascading.tap.hadoop.Hfs
Combined input format that uses the underlying individual input format to combine multiple files into a single split.

  public void testDanglingTail() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    Hfs tap = new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" );
    sources.put( "a", tap );
    sources.put( "b", tap );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );

View Full Code Here

  public void testExtraSource() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    Hfs tap = new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" );
    sources.put( "a", tap );
    sources.put( "b", tap );
    sources.put( "c", tap );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe group1 = new GroupBy( pipeA );
    Pipe group2 = new GroupBy( pipeB );


    Pipe merge = new GroupBy( "tail", Pipe.pipes( group1, group2 ), new Fields( "first", "second" ) );


    sinks.put( merge.getName(), new Hfs( new TextLine(), "output/path" ) );


    try
      {
      Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, merge );
      fail( "did not catch extra source tap" );

View Full Code Here

  public void testExtraSink() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    Hfs tap = new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" );
    sources.put( "a", tap );
    sources.put( "b", tap );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe group1 = new GroupBy( pipeA );
    Pipe group2 = new GroupBy( pipeB );


    Pipe merge = new GroupBy( "tail", Pipe.pipes( group1, group2 ), new Fields( "first", "second" ) );


    sinks.put( merge.getName(), new Hfs( new TextLine(), "output/path" ) );
    sinks.put( "c", new Hfs( new TextLine(), "output/path" ) );


    try
      {
      Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, merge );
      fail( "did not catch extra sink tap" );

View Full Code Here

  public void testBuffer() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );

View Full Code Here

  public void testBufferFail() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) );

View Full Code Here

  public void testBufferFail2() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );

View Full Code Here

    }


  @Test
  public void testErrorMessages() throws Exception
    {
    Tap source10 = new Hfs( new TextLine( new Fields( "num" ) ), "foo" );
    Tap source20 = new Hfs( new TextLine( new Fields( "num" ) ), "bar" );


    Map sources = new HashMap();


    sources.put( "source20", source20 );
    sources.put( "source101", source10 );
    sources.put( "source102", source10 );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), "baz", true );


    Pipe pipeNum20 = new Pipe( "source20" );
    Pipe pipeNum101 = new Pipe( "source101" );
    Pipe pipeNum102 = new Pipe( "source102" );

View Full Code Here

    splitMiddle( false, false );
    }


  private void splitMiddle( boolean before, boolean testTempReplaced )
    {
    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "lower" );
    Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "upper" );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );


    Scheme leftScheme = testTempReplaced ? new SequenceFile( new Fields( "num", "lower", "num2", "upper" ) ) : new TextLine( new Fields( "offset", "line" ), new Fields( "lower" ) );
    Tap sinkLeft = new Hfs( leftScheme, "/splitmiddle/left", SinkMode.REPLACE );


    Scheme rightScheme = testTempReplaced ? new SequenceFile( new Fields( "lower" ) ) : new TextLine( new Fields( "offset", "line" ), new Fields( "lower" ) );
    Tap sinkRight = new Hfs( rightScheme, "/splitmiddle/right", SinkMode.REPLACE );


    Map sinks = new HashMap();


    sinks.put( "left", sinkLeft );
    sinks.put( "right", sinkRight );

View Full Code Here

    }


  @Test
  public void testSourceIsSink()
    {
    Tap tap = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );


    Pipe pipe = new Pipe( "left" );


    try
      {

View Full Code Here

    }


  @Test
  public void testReplaceFail() throws Exception
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink = new Hfs( new TextLine( new Fields( "offset", "line" ), new Fields( "offset", "line2" ) ), "bar", true );


    Pipe pipe = new Pipe( "test" );


    Function parser = new RegexParser( new Fields( 0 ), "^[^ ]*" );
    pipe = new Each( pipe, new Fields( "line" ), parser, Fields.REPLACE );

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.tap.hadoop.Hfs$CombinedInputFormat

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.hadoop.FlowPlatformTest

cascading.flow.hadoop.MapReduceFlow

cascading.flow.hadoop.MapReduceFlowPlatformTest

cascading.flow.hadoop.planner.rule.scopeexpression.EquivalentTapsScopeExpression

cascading.flow.hadoop.util.HadoopMRUtil

cascading.flow.hadoop.util.HadoopUtil

cascading.lingual.platform.hadoop.HadoopDefaultFactory

cascading.lingual.platform.hadoop.HadoopPlatformBroker

cascading.lingual.platform.hadoop2.Hadoop2MR1DefaultFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.