Examples of cascading.scheme.hadoop.TextLine

cascading.scheme.hadoop.TextLine
A TextLine is a type of {@link cascading.scheme.Scheme} for plain text files. Files are broken intolines. Either line-feed or carriage-return are used to signal end of line.
By default, this scheme returns a {@link Tuple} with two fields, "offset" and "line".
Many of the constructors take both "sourceFields" and "sinkFields". sourceFields denote the field names to be used instead of the names "offset" and "line". sinkFields is a selector and is by default {@link Fields#ALL}. Any available field names can be given if only a subset of the incoming fields should be used.
If a {@link Fields} instance is passed on the constructor as sourceFields having only one field, the return tupleswill simply be the "line" value using the given field name.
Note that TextLine will concatenate all the Tuple values for the selected fields with a TAB delimiter before writing out the line.
Note sink compression is {@link Compress#DISABLE} by default. If {@code null} is passed to the constructorfor the compression value, it will remain disabled.
If any of the input files end with ".zip", an error will be thrown. *
By default, all text is encoded/decoded as UTF-8. This can be changed via the {@code charsetName} constructorargument.

    Properties properties = new Properties();
    AppProps.setApplicationJarClass( properties, Main.class );
    HadoopFlowConnector flowConnector = new HadoopFlowConnector( properties );


    // create taps for sources, sinks, traps
    Tap gisTap = new Hfs( new TextLine( new Fields( "line" ) ), gisPath );
    Tap metaTreeTap = new Hfs( new TextDelimited( true, "\t" ), metaTreePath );
    Tap metaRoadTap = new Hfs( new TextDelimited( true, "\t" ), metaRoadPath );
    Tap logsTap = new Hfs( new TextDelimited( true, "," ), logsPath );
    Tap trapTap = new Hfs( new TextDelimited( true, "\t" ), trapPath );
    Tap tsvTap = new Hfs( new TextDelimited( true, "\t" ), tsvPath );

View Full Code Here

    return copyConfiguration( properties, new JobConf() );
    }


  public static boolean removeStateFromDistCache( Configuration conf, String path ) throws IOException
    {
    return new Hfs( new TextLine(), path ).deleteResource( conf );
    }

View Full Code Here


  @Override
  public Tap getTextFile( Fields sourceFields, Fields sinkFields, String filename, SinkMode mode )
    {
    if( sourceFields == null )
      return new Hfs( new TextLine(), safeFileName( filename ), mode );


    return new Hfs( new TextLine( sourceFields, sinkFields ), safeFileName( filename ), mode );
    }

View Full Code Here

  public void testBufferFail2() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );

View Full Code Here

    }


  @Test
  public void testErrorMessages() throws Exception
    {
    Tap source10 = new Hfs( new TextLine( new Fields( "num" ) ), "foo" );
    Tap source20 = new Hfs( new TextLine( new Fields( "num" ) ), "bar" );


    Map sources = new HashMap();


    sources.put( "source20", source20 );
    sources.put( "source101", source10 );
    sources.put( "source102", source10 );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), "baz", true );


    Pipe pipeNum20 = new Pipe( "source20" );
    Pipe pipeNum101 = new Pipe( "source101" );
    Pipe pipeNum102 = new Pipe( "source102" );

View Full Code Here

    splitMiddle( false, false );
    }


  private void splitMiddle( boolean before, boolean testTempReplaced )
    {
    Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "lower" );
    Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "upper" );


    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );


    Scheme leftScheme = testTempReplaced ? new SequenceFile( new Fields( "num", "lower", "num2", "upper" ) ) : new TextLine( new Fields( "offset", "line" ), new Fields( "lower" ) );
    Tap sinkLeft = new Hfs( leftScheme, "/splitmiddle/left", SinkMode.REPLACE );


    Scheme rightScheme = testTempReplaced ? new SequenceFile( new Fields( "lower" ) ) : new TextLine( new Fields( "offset", "line" ), new Fields( "lower" ) );
    Tap sinkRight = new Hfs( rightScheme, "/splitmiddle/right", SinkMode.REPLACE );


    Map sinks = new HashMap();


    sinks.put( "left", sinkLeft );

View Full Code Here

    }


  @Test
  public void testSourceIsSink()
    {
    Tap tap = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );


    Pipe pipe = new Pipe( "left" );


    try
      {

View Full Code Here

    }


  @Test
  public void testReplaceFail() throws Exception
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink = new Hfs( new TextLine( new Fields( "offset", "line" ), new Fields( "offset", "line2" ) ), "bar", true );


    Pipe pipe = new Pipe( "test" );


    Function parser = new RegexParser( new Fields( 0 ), "^[^ ]*" );
    pipe = new Each( pipe, new Fields( "line" ), parser, Fields.REPLACE );

View Full Code Here

    }


  @Test
  public void testNestedProperties() throws IOException
    {
    Tap source = new Hfs( new TextLine( new Fields( "line" ) ), "/input" );


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third" ), "\\s" ), Fields.ALL );


    Tap sink = new Hfs( new TextLine(), "output", true );


    Properties defaultProperties = new Properties();


    defaultProperties.setProperty( "test.key", "test.value" );

View Full Code Here

    }


  @Test
  public void testEveryAfterJoin()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge1" );
    Tap source2 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge2" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe left = new Each( new Pipe( "left" ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );


    Fields[] fields = Fields.fields( new Fields( "offset" ), new Fields( "offset" ) );

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of cascading.scheme.hadoop.TextLine

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.hadoop.FlowPlatformTest

cascading.flow.hadoop.MapReduceFlowPlatformTest

cascading.flow.hadoop.util.HadoopMRUtil

cascading.flow.hadoop.util.HadoopUtil

cascading.lingual.platform.hadoop.HadoopPlatformBroker

cascading.lingual.platform.hadoop2.Hadoop2MR1PlatformBroker

cascading.platform.hadoop.BaseHadoopPlatform

cascading.tap.hadoop.HadoopMR1TapPlatformTest

cascading.tap.hadoop.HadoopTapPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.