Examples of cascading.scheme.hadoop.TextLine

cascading.scheme.hadoop.TextLine
A TextLine is a type of {@link cascading.scheme.Scheme} for plain text files. Files are broken intolines. Either line-feed or carriage-return are used to signal end of line.
By default, this scheme returns a {@link Tuple} with two fields, "offset" and "line".
Many of the constructors take both "sourceFields" and "sinkFields". sourceFields denote the field names to be used instead of the names "offset" and "line". sinkFields is a selector and is by default {@link Fields#ALL}. Any available field names can be given if only a subset of the incoming fields should be used.
If a {@link Fields} instance is passed on the constructor as sourceFields having only one field, the return tupleswill simply be the "line" value using the given field name.
Note that TextLine will concatenate all the Tuple values for the selected fields with a TAB delimiter before writing out the line.
Note sink compression is {@link Compress#DISABLE} by default. If {@code null} is passed to the constructorfor the compression value, it will remain disabled.
If any of the input files end with ".zip", an error will be thrown. *
By default, all text is encoded/decoded as UTF-8. This can be changed via the {@code charsetName} constructorargument.

  public void testSameSourceForBranch() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "a", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" ) );


    Pipe pipeA = new Pipe( "a" );


    Pipe group1 = new GroupBy( "a1", pipeA, Fields.FIRST );
    Pipe group2 = new GroupBy( "a2", pipeA, Fields.FIRST );


    Pipe merge = new GroupBy( "tail", Pipe.pipes( group1, group2 ), new Fields( "first", "second" ) );


    sinks.put( merge.getName(), new Hfs( new TextLine(), "output/path" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, merge );


    assertEquals( "not equal: steps.size()", 3, flow.getFlowSteps().size() );
    }

View Full Code Here

  public void testSameTaps() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    Hfs tap = new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" );
    sources.put( "a", tap );
    sources.put( "b", tap );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe group1 = new GroupBy( pipeA );
    Pipe group2 = new GroupBy( pipeB );


    Pipe merge = new GroupBy( "tail", Pipe.pipes( group1, group2 ), new Fields( "first", "second" ) );


    sinks.put( merge.getName(), new Hfs( new TextLine(), "output/path" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, merge );


    assertEquals( "not equal: steps.size()", 3, flow.getFlowSteps().size() );
    }

View Full Code Here

  public void testDanglingHead() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    Hfs source = new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" );
    sources.put( "a", source );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe group1 = new GroupBy( pipeA );
    Pipe group2 = new GroupBy( pipeB );


    Pipe merge = new GroupBy( "tail", Pipe.pipes( group1, group2 ), new Fields( "first", "second" ) );


    sinks.put( merge.getName(), new Hfs( new TextLine(), "output/path" ) );


    try
      {
      Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, merge );
      fail( "did not catch missing source tap" );

View Full Code Here

  public void testDanglingTail() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    Hfs tap = new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" );
    sources.put( "a", tap );
    sources.put( "b", tap );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );

View Full Code Here

  public void testExtraSource() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    Hfs tap = new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" );
    sources.put( "a", tap );
    sources.put( "b", tap );
    sources.put( "c", tap );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe group1 = new GroupBy( pipeA );
    Pipe group2 = new GroupBy( pipeB );


    Pipe merge = new GroupBy( "tail", Pipe.pipes( group1, group2 ), new Fields( "first", "second" ) );


    sinks.put( merge.getName(), new Hfs( new TextLine(), "output/path" ) );


    try
      {
      Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, merge );
      fail( "did not catch extra source tap" );

View Full Code Here

  public void testExtraSink() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    Hfs tap = new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" );
    sources.put( "a", tap );
    sources.put( "b", tap );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe group1 = new GroupBy( pipeA );
    Pipe group2 = new GroupBy( pipeB );


    Pipe merge = new GroupBy( "tail", Pipe.pipes( group1, group2 ), new Fields( "first", "second" ) );


    sinks.put( merge.getName(), new Hfs( new TextLine(), "output/path" ) );
    sinks.put( "c", new Hfs( new TextLine(), "output/path" ) );


    try
      {
      Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, merge );
      fail( "did not catch extra sink tap" );

View Full Code Here

  public void testBuffer() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );

View Full Code Here

  public void testBufferFail() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) );

View Full Code Here

  public void testCombinedHfs() throws Exception
    {
    getPlatform().copyFromLocal( inputFileLower );
    getPlatform().copyFromLocal( inputFileUpper );


    Hfs sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputFileLower );
    Hfs sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputFileUpper );


    // create a CombinedHfs instance on these files
    Tap source = new MultiSourceTap<Hfs, JobConf, RecordReader>( sourceLower, sourceUpper );


    FlowProcess<JobConf> process = getPlatform().getFlowProcess();

View Full Code Here


    LOG.info( "writing step state to dist cache, too large for job conf, size: {}", stepState.length() );


    String statePath = Hfs.getTempPath( conf ) + "/" + kind + "-state-" + id;


    Hfs temp = new Hfs( new TextLine(), statePath, SinkMode.REPLACE );


    try
      {
      TupleEntryCollector writer = temp.openForWrite( new HadoopFlowProcess( conf ) );

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of cascading.scheme.hadoop.TextLine

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.hadoop.FlowPlatformTest

cascading.flow.hadoop.MapReduceFlowPlatformTest

cascading.flow.hadoop.util.HadoopMRUtil

cascading.flow.hadoop.util.HadoopUtil

cascading.lingual.platform.hadoop.HadoopPlatformBroker

cascading.lingual.platform.hadoop2.Hadoop2MR1PlatformBroker

cascading.platform.hadoop.BaseHadoopPlatform

cascading.tap.hadoop.HadoopMR1TapPlatformTest

cascading.tap.hadoop.HadoopTapPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.