Examples of cascading.operation.expression.ExpressionFunction

cascading.operation.expression.ExpressionFunction
anino.net/">Janino compiler.
Specifically this function uses the {@link ExpressionEvaluator}, thus the syntax from that class is inherited here.
An expression may use field names directly as parameters in the expression, or field positions with the syntax "$n", where n is an integer.
Given an argument tuple with the fields "a" and "b", the following expression returns true:
a + b == $0 + $1

Further, the types of the tuple elements will be coerced into the given parameterTypes. Regardless of the actual tuple element values, they will be converted to the types expected by the expression.
Field names used in the expression should be valid Java variable names; for example, '+' or '-' are not allowed. Also the use of a field name that begins with an upper-case character is likely to fail and should be avoided.

    Pipe tfidfPipe = new CoGroup( tfPipe, tf_token, idfPipe, df_token );


    // calculate the TF-IDF weights, per token, per document
    Fields tfidf = new Fields( "tfidf" );
    String expression = "(double) tf_count * Math.log( (double) n_docs / ( 1.0 + df_count ) )";
    ExpressionFunction tfidfExpression = new ExpressionFunction( tfidf, expression, Double.class );
    Fields tfidfArguments = new Fields( "tf_count", "df_count", "n_docs" );
    tfidfPipe = new Each( tfidfPipe, tfidfArguments, tfidfExpression, Fields.ALL );


    fieldSelector = new Fields( "tf_token", "doc_id", "tfidf" );
    tfidfPipe = new Retain( tfidfPipe, fieldSelector );

View Full Code Here

    {
    super( previous );


    String expression = "java.util.UUID.randomUUID().toString()";


    ExpressionFunction expressionFunction = new ExpressionFunction( declaredFields, expression )
    {
    @Override
    public boolean isSafe()
      {
      return false;

View Full Code Here

    tsCountPipe = new GroupBy( tsCountPipe, new Fields( "ts" ) );
    tsCountPipe = new Every( tsCountPipe, Fields.GROUP, new Count() );


    // apply expression to create a timestamp with 'minute' granularity
    // declares field "tm"
    Pipe tmPipe = new Each( tsPipe, new ExpressionFunction( new Fields( "tm" ), "ts - (ts % (60 * 1000))", long.class ) );


    // name the per minute assembly and split on tmPipe
    Pipe tmCountPipe = new Pipe( "tmCount", tmPipe );
    tmCountPipe = new GroupBy( tmCountPipe, new Fields( "tm" ) );
    tmCountPipe = new Every( tmCountPipe, Fields.GROUP, new Count() );

View Full Code Here

    regex = "^([\\w\\s]+).*$";
    int[] speciesGroups = { 1 };
    parser = new RegexParser( new Fields( "scrub_species" ), regex, speciesGroups );
    treePipe = new Each( treePipe, new Fields( "raw_species" ), parser, Fields.ALL );
    String expression = "scrub_species.trim().toLowerCase()";
    ExpressionFunction exprFunc = new ExpressionFunction( new Fields( "tree_species" ), expression, String.class );
    treePipe = new Each( treePipe, new Fields( "scrub_species" ), exprFunc, Fields.ALL );


    // join with tree metadata
    Pipe metaTreePipe = new Pipe( "meta_tree" );
    treePipe = new HashJoin( treePipe, new Fields( "tree_species" ), metaTreePipe, new Fields( "species" ), new InnerJoin() );

View Full Code Here

    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count( new Fields( "count1" ) ) );
    pipe = new Every( pipe, new Count( new Fields( "count2" ) ) );


    pipe = new Each( pipe, new Fields( "count1", "count2" ), new ExpressionFunction( new Fields( "sum" ), "count1 + count2", int.class ), Fields.ALL );


    Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache );
    Tap sink = getPlatform().getTextFile( getOutputPath( "chaineach" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );

View Full Code Here


    Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );


    pipeLower = new Each( pipeLower, new Fields( "num" ), new ExpressionFunction( Fields.ARGS, "Integer.parseInt( num )", String.class ), Fields.REPLACE );


    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );


    Fields num = new Fields( "num" );
    num.setComparator( "num", new AllComparator() );

View Full Code Here

    Tap sink = getPlatform().getDelimitedFile( new Fields( "notnull", "null" ), "\t",
      new Class[]{Integer.TYPE, Integer.TYPE}, getOutputPath( "countnullnotnull" ), SinkMode.REPLACE );


    Pipe pipe = new Pipe( "count" );


    ExpressionFunction function = new ExpressionFunction( Fields.ARGS, "\"c\".equals($0) ? null : $0", String.class );
    pipe = new Each( pipe, new Fields( "char" ), function, Fields.REPLACE );


    CountBy countNotNull = new CountBy( new Fields( "char" ), new Fields( "notnull" ), CountBy.Include.NO_NULLS );
    CountBy countNull = new CountBy( new Fields( "char" ), new Fields( "null" ), CountBy.Include.ONLY_NULLS );

View Full Code Here


    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileApache );


    Pipe pipe = new Pipe( "keyvalue" );


    pipe = new Each( pipe, new Fields( "offset" ), new ExpressionFunction( Fields.ARGS, "new org.apache.hadoop.io.LongWritable($0)", long.class ), Fields.REPLACE );
    pipe = new Each( pipe, new Fields( "line" ), new ExpressionFunction( Fields.ARGS, "new org.apache.hadoop.io.Text($0)", String.class ), Fields.REPLACE );


    Tap tapKeyValue = new Hfs( new WritableSequenceFile( new Fields( "offset", "line" ), LongWritable.class, Text.class ), getOutputPath( "keyvalue" ), SinkMode.REPLACE );
    Tap tapKey = new Hfs( new WritableSequenceFile( new Fields( "offset" ), LongWritable.class, null ), getOutputPath( "key" ), SinkMode.REPLACE );
    Tap tapValue = new Hfs( new WritableSequenceFile( new Fields( "line" ), Text.class ), getOutputPath( "value" ), SinkMode.REPLACE );

View Full Code Here

      new Class[]{String.class, Integer.TYPE}, getOutputPath( "mergecount" ), SinkMode.REPLACE );


    Pipe lhsPipe = new Pipe( "count-lhs" );
    Pipe rhsPipe = new Pipe( "count-rhs" );


    rhsPipe = new Each( rhsPipe, new Fields( "char" ), new ExpressionFunction( Fields.ARGS, "$0.toLowerCase()", String.class ), Fields.REPLACE );


    Pipe countPipe = new CountBy( Pipe.pipes( lhsPipe, rhsPipe ), new Fields( "char" ), new Fields( "count" ), 2 );


    Map<String, Tap> tapMap = Cascades.tapsMap( Pipe.pipes( lhsPipe, rhsPipe ), Tap.taps( lhs, rhs ) );

View Full Code Here

    Tap sink = getPlatform().getDelimitedFile( new Fields( "char", "sum" ), "\t",
      new Class[]{String.class, Integer.class}, getOutputPath( "sumnulls" ), SinkMode.REPLACE );


    Pipe pipe = new Pipe( "sum" );


    ExpressionFunction function = new ExpressionFunction( Fields.ARGS, "5 == $0 ? null : $0", Integer.class );
    pipe = new Each( pipe, new Fields( "num" ), function, Fields.REPLACE );


    // Long.class denotes return null for null, not zero
    pipe = new SumBy( pipe, new Fields( "char" ), new Fields( "num" ), new Fields( "sum" ), Integer.class, 2 );

View Full Code Here

0 1

TOP

Related Classes of cascading.operation.expression.ExpressionFunction

cascading.FieldedPipesPlatformTest

cascading.JoinFieldedPipesPlatformTest

cascading.pattern.ensemble.function.InsertGUID

cascading.pipe.assembly.AssemblyHelpersPlatformTest

cascading.RegressionPipesPlatformTest

cascading.scheme.hadoop.WritableSequenceFilePlatformTest

copa.Main

impatient.Main

loganalysis.Main

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.