Examples of cascading.scheme.hadoop.TextLine

cascading.scheme.hadoop.TextLine
A TextLine is a type of {@link cascading.scheme.Scheme} for plain text files. Files are broken intolines. Either line-feed or carriage-return are used to signal end of line.
By default, this scheme returns a {@link Tuple} with two fields, "offset" and "line".
Many of the constructors take both "sourceFields" and "sinkFields". sourceFields denote the field names to be used instead of the names "offset" and "line". sinkFields is a selector and is by default {@link Fields#ALL}. Any available field names can be given if only a subset of the incoming fields should be used.
If a {@link Fields} instance is passed on the constructor as sourceFields having only one field, the return tupleswill simply be the "line" value using the given field name.
Note that TextLine will concatenate all the Tuple values for the selected fields with a TAB delimiter before writing out the line.
Note sink compression is {@link Compress#DISABLE} by default. If {@code null} is passed to the constructorfor the compression value, it will remain disabled.
If any of the input files end with ".zip", an error will be thrown. *
By default, all text is encoded/decoded as UTF-8. This can be changed via the {@code charsetName} constructorargument.

//    }


  @Test
  public void testMerge2()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge1" );
    Tap source2 = new Hfs( new SequenceFile( new Fields( "offset", "line" ) ), "foo/merge2" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe left = new Each( new Pipe( "left" ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );


    Pipe merge = new GroupBy( "merge", Pipe.pipes( left, right ), new Fields( "offset" ) );

View Full Code Here

  @Test
  public void testSimpleGroup() throws Exception
    {
    getPlatform().copyFromLocal( inputFileApache );


    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileApache );


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );

View Full Code Here

  @Test
  public void testSimpleGroupOnBytes() throws Exception
    {
    getPlatform().copyFromLocal( inputFileApache );


    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileApache );


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );

View Full Code Here

   * Tests the case where the same source is split, then re-merged
   */
  @Test
  public void testMergeSameSourceSplit()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge1" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe head = new Pipe( "source" );


    head = new Each( head, new Fields( "line" ), new ExpressionFilter( "line.length() != 0", String.class ) );

View Full Code Here

    }


  @Test
  public void testCoGroupAroundCoGroup() throws Exception
    {
    Tap source10 = new Hfs( new TextLine( new Fields( "num" ) ), "foo" );
    Tap source20 = new Hfs( new TextLine( new Fields( "num" ) ), "bar" );


    Map sources = new HashMap();


    sources.put( "source20", source20 );
    sources.put( "source101", source10 );
    sources.put( "source102", source10 );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), "baz", true );


    Pipe pipeNum20 = new Pipe( "source20" );
    Pipe pipeNum101 = new Pipe( "source101" );
    Pipe pipeNum102 = new Pipe( "source102" );

View Full Code Here

  public void testCoGroupWritableAsKeyValue() throws Exception
    {
    getPlatform().copyFromLocal( inputFileLower );
    getPlatform().copyFromLocal( inputFileUpper );


    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower );
    Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileUpper );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );

View Full Code Here

  public void testCoGroupBytesWritableAsKeyValue() throws Exception
    {
    getPlatform().copyFromLocal( inputFileLower );
    getPlatform().copyFromLocal( inputFileUpper );


    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower );
    Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileUpper );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );


    Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine( new Fields( "line" ) ), getOutputPath( "byteswritablekeyvalue" ), SinkMode.REPLACE );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    pipeLower = new Each( pipeLower, new Fields( "char" ), new ReplaceAsBytes( new Fields( "char" ) ), Fields.REPLACE );


    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );

View Full Code Here

    }


  @Test
  public void testCoGroupAroundCoGroupOptimized() throws Exception
    {
    Tap source10 = new Hfs( new TextLine( new Fields( "num" ) ), "foo" );
    Tap source20 = new Hfs( new TextLine( new Fields( "num" ) ), "bar" );


    Map sources = new HashMap();


    sources.put( "source20", source20 );
    sources.put( "source101", source10 );
    sources.put( "source102", source10 );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), "baz", true );


    Pipe pipeNum20 = new Pipe( "source20" );
    Pipe pipeNum101 = new Pipe( "source101" );
    Pipe pipeNum102 = new Pipe( "source102" );

View Full Code Here

  public void testCoGroupSpillCustomWritable() throws Exception
    {
    getPlatform().copyFromLocal( inputFileLower );
    getPlatform().copyFromLocal( inputFileUpper );


    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower );
    Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileUpper );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );

View Full Code Here

    throws IOException
    {
    getPlatform().copyFromLocal( inputFileLower );
    getPlatform().copyFromLocal( inputFileUpper );


    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower );
    Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileUpper );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of cascading.scheme.hadoop.TextLine

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.hadoop.FlowPlatformTest

cascading.flow.hadoop.MapReduceFlowPlatformTest

cascading.flow.hadoop.util.HadoopMRUtil

cascading.flow.hadoop.util.HadoopUtil

cascading.lingual.platform.hadoop.HadoopPlatformBroker

cascading.lingual.platform.hadoop2.Hadoop2MR1PlatformBroker

cascading.platform.hadoop.BaseHadoopPlatform

cascading.tap.hadoop.HadoopMR1TapPlatformTest

cascading.tap.hadoop.HadoopTapPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.