ExpressionFunction exprFunc = new ExpressionFunction( new Fields( "tree_species" ), expression, String.class );
treePipe = new Each( treePipe, new Fields( "scrub_species" ), exprFunc, Fields.ALL );
// join with tree metadata
Pipe metaTreePipe = new Pipe( "meta_tree" );
treePipe = new HashJoin( treePipe, new Fields( "tree_species" ), metaTreePipe, new Fields( "species" ), new InnerJoin() );
treePipe = new Rename( treePipe, new Fields( "blurb" ), new Fields( "tree_name" ) );
regex = "^(\\S+),(\\S+),(\\S+)\\s*$";
int[] gpsGroups = { 1, 2, 3 };
parser = new RegexParser( new Fields( "tree_lat", "tree_lng", "tree_alt" ), regex, gpsGroups );
treePipe = new Each( treePipe, new Fields( "geo" ), parser, Fields.ALL );
// determine a tree geohash
Fields geohashArguments = new Fields( "tree_lat", "tree_lng" );
treePipe = new Each( treePipe, geohashArguments, new GeoHashFunction( new Fields( "tree_geohash" ), 6 ), Fields.ALL );
Fields fieldSelector = new Fields( "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_geohash" );
treePipe = new Retain( treePipe, fieldSelector );
// parse the "road" output
Pipe roadPipe = new Pipe( "road", tsvCheck );
regex = "^\\s+Sequence\\:.*\\s+Year Constructed\\:\\s+(\\d+)\\s+Traffic Count\\:\\s+(\\d+)\\s+Traffic Index\\:\\s+(\\w.*\\w)\\s+Traffic Class\\:\\s+(\\w.*\\w)\\s+Traffic Date.*\\s+Paving Length\\:\\s+(\\d+)\\s+Paving Width\\:\\s+(\\d+)\\s+Paving Area\\:\\s+(\\d+)\\s+Surface Type\\:\\s+(\\w.*\\w)\\s+Surface Thickness.*\\s+Bike Lane\\:\\s+(\\w+)\\s+Bus Route\\:\\s+(\\w+)\\s+Truck Route\\:\\s+(\\w+)\\s+Remediation.*$";
roadPipe = new Each( roadPipe, new Fields( "misc" ), new RegexFilter( regex ) );
Fields roadFields = new Fields( "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route" );
int[] roadGroups = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
parser = new RegexParser( roadFields, regex, roadGroups );
roadPipe = new Each( roadPipe, new Fields( "misc" ), parser, Fields.ALL );
// join with road metadata
Pipe metaRoadPipe = new Pipe( "meta_road" );
roadPipe = new HashJoin( roadPipe, new Fields( "surface_type" ), metaRoadPipe, new Fields( "pavement_type" ), new InnerJoin() );
roadPipe = new Rename( roadPipe, new Fields( "blurb" ), new Fields( "road_name" ) );
// estimate albedo based on the road surface age and pavement type
Fields albedoArguments = new Fields( "year_construct", "albedo_new", "albedo_worn" );
roadPipe = new Each( roadPipe, albedoArguments, new AlbedoFunction( new Fields( "albedo" ), 2002 ), Fields.ALL );
// generate road segments, with midpoint, y=mx+b, and road_geohash for each
Fields segmentArguments = new Fields( "geo" );
Fields segmentResults = new Fields( "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "lat_mid", "lng_mid" );
roadPipe = new Each( roadPipe, segmentArguments, new RoadSegmentFunction( segmentResults ), Fields.ALL );
geohashArguments = new Fields( "lat_mid", "lng_mid" );
roadPipe = new Each( roadPipe, geohashArguments, new GeoHashFunction( new Fields( "road_geohash" ), 6 ), Fields.ALL );
fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "road_geohash" );
roadPipe = new Retain( roadPipe, fieldSelector );
// join the tree and road pipes to estimate shade
Pipe shadePipe = new Pipe( "shade", roadPipe );
shadePipe = new CoGroup( shadePipe, new Fields( "road_geohash" ), treePipe, new Fields( "tree_geohash" ), new InnerJoin() );
// calculate a rough estimate for distance from tree to road, then filter for "< ~1 block"
Fields treeDistArguments = new Fields( "tree_lat", "tree_lng", "lat0", "lng0", "lat1", "lng1" );
Fields tree_dist = new Fields( "tree_dist" );
shadePipe = new Each( shadePipe, treeDistArguments, new TreeDistanceFunction( tree_dist ), Fields.ALL );
ExpressionFilter distFilter = new ExpressionFilter( "tree_dist > 25.0", Double.class );
shadePipe = new Each( shadePipe, tree_dist, distFilter );
// checkpoint this (big) calculation too
fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "lat1", "lng1", "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_dist", "tree_geohash" );
shadePipe = new Retain( shadePipe, fieldSelector );
shadePipe = new GroupBy( shadePipe, new Fields( "tree_name" ), new Fields( "tree_dist" ) );
Checkpoint shadeCheck = new Checkpoint( "shade", shadePipe );
// determine the geohash for GPS tracks log events
Pipe logsPipe = new Pipe( "logs" );
geohashArguments = new Fields( "lat", "lng" );
logsPipe = new Each( logsPipe, geohashArguments, new GeoHashFunction( new Fields( "gps_geohash" ), 6 ), Fields.ALL );
// prepare data for recommendations
// NB: RHS is large given the sample data, but in practice the logs on the LHS could be much larger
Pipe recoPipe = new Pipe( "reco", logsPipe );
recoPipe = new CoGroup( recoPipe, new Fields( "gps_geohash" ), shadeCheck, new Fields( "tree_geohash" ), new InnerJoin() );
// connect the taps, pipes, etc., into a flow
FlowDef flowDef = FlowDef.flowDef()
.setName( "copa" )
.addSource( gisPipe, gisTap )