Examples of cascading.scheme.local.TextLine

cascading.scheme.local.TextLine
A TextLine is a type of {@link cascading.scheme.Scheme} for plain text files. Files are broken intolines. Either line-feed or carriage-return are used to signal end of line.
By default, this scheme returns a {@link cascading.tuple.Tuple} with two fields, "num" and "line". Where "num"is the line number for "line".
Many of the constructors take both "sourceFields" and "sinkFields". sourceFields denote the field names to be used instead of the names "num" and "line". sinkFields is a selector and is by default {@link Fields#ALL}. Any available field names can be given if only a subset of the incoming fields should be used.
If a {@link Fields} instance is passed on the constructor as sourceFields having only one field, the return tupleswill simply be the "line" value using the given field name.
Note that TextLine will concatenate all the Tuple values for the selected fields with a TAB delimiter before writing out the line.
By default, all text is encoded/decoded as UTF-8. This can be changed via the {@code charsetName} constructorargument.

        //pipe = new Each(pipe, new Fields("name"), AssertionLevel.STRICT, new AssertNotNull());
        pipe = new GroupBy(pipe);
        pipe = new Every(pipe, new Count());


        // print out
        Tap out = new OutputStreamTap(new TextLine(), OUT);
        build(cfg(), in, out, pipe);
    }

View Full Code Here

        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertNotNull());
        pipe = new GroupBy(pipe);
        pipe = new Every(pipe, new Count());


        // print out
        Tap out = new OutputStreamTap(new TextLine(), OUT);
        build(cfg(), in, out, pipe);
    }

View Full Code Here

        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertNotNull());
        pipe = new GroupBy(pipe);
        pipe = new Every(pipe, new Count());


        // print out
        Tap out = new OutputStreamTap(new TextLine(), OUT);
        build(cfg(), in, out, pipe);
    }

View Full Code Here

        pipe = new GroupBy(pipe);
        pipe = new Every(pipe, new Count());


        // print out
        // print out
        Tap out = new OutputStreamTap(new TextLine(), OUT);
        Properties cfg = cfg();
        cfg.setProperty("es.mapping.names", "url:address");
        build(cfg, in, out, pipe);
    }

View Full Code Here

    private void build(Properties props, Tap in, Tap out, Pipe pipe) {
        StatsUtils.proxy(new LocalFlowConnector(props).connect(in, out, pipe)).complete();
    }


    private Tap sourceTap() {
        return new FileTap(new TextLine(new Fields("line")), TestUtils.sampleArtistsJson());
    }

View Full Code Here

        //pipe = new Each(pipe, new Fields("name"), AssertionLevel.STRICT, new AssertNotNull());
        pipe = new GroupBy(pipe);
        pipe = new Every(pipe, new Count());


        // print out
        Tap out = new OutputStreamTap(new TextLine(), OUT);
        build(cfg(), in, out, pipe);
    }

View Full Code Here

        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, new FilterNotNull());
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(2));


        // print out
        Tap out = new OutputStreamTap(new TextLine(), OUT);
        build(cfg, in, out, pipe);
    }

View Full Code Here

            pipe = new Each(pipe, new UrlLengthener(fetcher));
            pipe = new Each(pipe, new Debug());


            BixoPlatform platform = new BixoPlatform(LengthenUrlsTool.class, Platform.Local);
            BasePath filePath = platform.makePath(filename);
            TextLine textLineLocalScheme = new TextLine(new Fields("url"));
            Tap sourceTap = platform.makeTap(textLineLocalScheme, filePath, SinkMode.KEEP);
            SinkTap sinkTap = new NullSinkTap(new Fields("url"));
            
            FlowConnector flowConnector = platform.makeFlowConnector();
            Flow flow = flowConnector.connect(sourceTap, sinkTap, pipe);

View Full Code Here


  @Override
  public Tap getTextFile( Fields sourceFields, Fields sinkFields, String filename, SinkMode mode )
    {
    if( sourceFields == null )
      return new FileTap( new TextLine(), filename, mode );


    return new FileTap( new TextLine( sourceFields, sinkFields ), filename, mode );
    }

View Full Code Here

    String lines = "line1\nline2\n";
    System.setIn( new ByteArrayInputStream( lines.getBytes() ) );
    ByteArrayOutputStream output = new ByteArrayOutputStream();
    System.setOut( new PrintStream( output ) );


    Tap source = new StdInTap( new TextLine( new Fields( "line" ) ) );
    Tap sink = new StdOutTap( new TextLine( new Fields( "line" ) ) );


    Pipe pipe = new Pipe( "io" );


    Flow flow = new LocalFlowConnector().connect( source, sink, pipe );

View Full Code Here

0 1

TOP

Related Classes of cascading.scheme.local.TextLine

bixo.tools.LengthenUrlsTool

cascading.lingual.platform.local.LocalPlatformBroker

cascading.platform.local.LocalPlatform

cascading.tap.local.LocalTapPlatformTest

cascading.tuple.Fields

cascading.tuple.TupleEntry

org.elasticsearch.hadoop.integration.cascading.AbstractCascadingLocalJsonSaveTest

org.elasticsearch.hadoop.integration.cascading.AbstractCascadingLocalJsonSearchTest

org.elasticsearch.hadoop.integration.cascading.AbstractCascadingLocalSearchTest

java.io.OutputStreamWriter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.