Package cascading.tap.hadoop

Examples of cascading.tap.hadoop.Hfs$CombinedInputFormat

381
382
383
384
385
386
387
388
389
390
391
392
393
    }

  @Test
  public void testSplitOnNonSafeOperations()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );

    Pipe pipe = new Pipe( "split" );

    // this operation is not safe
    pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );
View Full Code Here
416
417
418
419
420
421
422
423
424
425
426
427
428
    }

  @Test
  public void testSplitOnNonSafeOperationsSimple()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );

    Pipe pipe = new Pipe( "split" );

    // this operation is not safe
    pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );
View Full Code Here
450
451
452
453
454
455
456
457
458
459
460
461
462
463
  // verify unsafe splits happen when splitting on a pipe
  @Test
  public void testSplitOnNonSafeOperations2()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );
    Tap sink3 = new Hfs( new TextLine(), "foo/split3", true );

    Pipe pipe = new Pipe( "split" );

    // this operation is not safe
    pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );
View Full Code Here
494
495
496
497
498
499
500
501
502
503
504
505
506
   * We previously would push the each to the next step, but if there is already data being written, save the cpu.
   */
  @Test
  public void testSplitComplex()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );

    Pipe pipe = new Pipe( "split" );

    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );

View Full Code Here
549
550
551
552
553
554
555
556
557
558
559
560
561
  /** same as splitComplex, except pipe/branch naming is after the Each, not before */
  @Test
  public void testSplitComplex2()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );

    Pipe pipe = new Pipe( "split" );

    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );

View Full Code Here
605
606
607
608
609
610
611
612
613
614
615
616
617
618
    }

  @Test
  public void testMerge()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge1" );
    Tap source2 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge2" );

    Tap sink = new Hfs( new TextLine(), "foo" );

    Pipe left = new Each( new Pipe( "left" ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );

    Pipe merge = new GroupBy( "merge", Pipe.pipes( left, right ), new Fields( "offset" ) );
View Full Code Here
632
633
634
635
636
637
638
639
640
641
642
643
644
645
    }

  @Test
  public void testDupeSource()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );
    Tap source2 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );

    Tap sink = new Hfs( new TextLine(), "foo" );

    Pipe left = new Each( new Pipe( "left" ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );
    right = new Each( right, new Fields( "line" ), new RegexFilter( ".*192.*" ) );
    right = new Each( right, new Fields( "line" ), new RegexFilter( ".*192.*" ) );
View Full Code Here
662
663
664
665
666
667
668
669
670
671
672
673
674
    }

  @Test
  public void testDupeSourceRepeat()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );

    Tap sink = new Hfs( new TextLine(), "foo" );

    Pipe pipe = new Pipe( "pipe" );

    Pipe merge = new CoGroup( "cogroup", pipe, new Fields( "offset" ), 1, Fields.size( 4 ) );
View Full Code Here
686
687
688
689
690
691
692
693
694
695
696
697
698
    }

  @Test
  public void testDupeSource2()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );

    Tap sink = new Hfs( new TextLine(), "foo" );

    Pipe left = new Pipe( "left" );
    Pipe right = new Pipe( "right" );

    Pipe merge = new CoGroup( "cogroup", left, new Fields( "offset" ), right, new Fields( "offset" ), Fields.size( 4 ) );
View Full Code Here
717
718
719
720
721
722
723
724
725
726
727
728
729
730
    }

  @Test
  public void testDupeSource3()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );
    Tap source2 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "bar/merge" );

    Tap sink = new Hfs( new TextLine(), "foo" );

    Pipe left = new Pipe( "left" );
    Pipe middle = new Pipe( "middle" );
    Pipe right = new Pipe( "right" );
View Full Code Here
TOP

Related Classes of cascading.tap.hadoop.Hfs$CombinedInputFormat

  • cascading.flow.hadoop.BuildJobsHadoopPlatformTest
  • cascading.flow.hadoop.FlowPlatformTest
  • cascading.flow.hadoop.MapReduceFlow
  • cascading.flow.hadoop.MapReduceFlowPlatformTest
  • cascading.flow.hadoop.planner.rule.scopeexpression.EquivalentTapsScopeExpression
  • cascading.flow.hadoop.util.HadoopMRUtil
  • cascading.flow.hadoop.util.HadoopUtil
  • cascading.lingual.platform.hadoop.HadoopDefaultFactory
  • cascading.lingual.platform.hadoop.HadoopPlatformBroker
  • cascading.lingual.platform.hadoop2.Hadoop2MR1DefaultFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.