Examples of Checkpoint


Examples of cascading.pipe.Checkpoint

    int[] gisGroups = { 1, 2, 3, 4 };
    RegexParser parser = new RegexParser( fieldDeclaration, regex, gisGroups );
    Pipe gisPipe = new Each( new Pipe( "gis" ), new Fields( "line" ), parser );

    // checkpoint the cleaned-up GIS data
    Checkpoint tsvCheck = new Checkpoint( "tsv", gisPipe );

    // parse the "park" output
    Pipe parkPipe = new Pipe( "park", tsvCheck );
    regex = "^\\s+Community Type\\:\\s+Park.*$";
    parkPipe = new Each( parkPipe, new Fields( "misc" ), new RegexFilter( regex ) );

    // parse the "tree" output
    Pipe treePipe = new Pipe( "tree", tsvCheck );
    regex = "^\\s+Private\\:\\s+(\\S+)\\s+Tree ID\\:\\s+(\\d+)\\s+.*Situs Number\\:\\s+(\\d+)\\s+Tree Site\\:\\s+(\\d+)\\s+Species\\:\\s+(\\S.*\\S)\\s+Source.*$";
    treePipe = new Each( treePipe, new Fields( "misc" ), new RegexFilter( regex ) );

    Fields treeFields = new Fields( "priv", "tree_id", "situs", "tree_site", "raw_species" );
    int[] treeGroups = { 1, 2, 3, 4, 5 };
    parser = new RegexParser( treeFields, regex, treeGroups );
    treePipe = new Each( treePipe, new Fields( "misc" ), parser, Fields.ALL );

    // scrub "species" as a primary key
    regex = "^([\\w\\s]+).*$";
    int[] speciesGroups = { 1 };
    parser = new RegexParser( new Fields( "scrub_species" ), regex, speciesGroups );
    treePipe = new Each( treePipe, new Fields( "raw_species" ), parser, Fields.ALL );
    String expression = "scrub_species.trim().toLowerCase()";
    ExpressionFunction exprFunc = new ExpressionFunction( new Fields( "tree_species" ), expression, String.class );
    treePipe = new Each( treePipe, new Fields( "scrub_species" ), exprFunc, Fields.ALL );

    // join with tree metadata
    Pipe metaTreePipe = new Pipe( "meta_tree" );
    treePipe = new HashJoin( treePipe, new Fields( "tree_species" ), metaTreePipe, new Fields( "species" ), new InnerJoin() );
    treePipe = new Rename( treePipe, new Fields( "blurb" ), new Fields( "tree_name" ) );

    regex = "^(\\S+),(\\S+),(\\S+)\\s*$";
    int[] gpsGroups = { 1, 2, 3 };
    parser = new RegexParser( new Fields( "tree_lat", "tree_lng", "tree_alt" ), regex, gpsGroups );
    treePipe = new Each( treePipe, new Fields( "geo" ), parser, Fields.ALL );

    // determine a tree geohash
    Fields geohashArguments = new Fields( "tree_lat", "tree_lng" );
    treePipe = new Each( treePipe, geohashArguments, new GeoHashFunction( new Fields( "tree_geohash" ), 6 ), Fields.ALL );

    Fields fieldSelector = new Fields( "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_geohash" );
    treePipe = new Retain( treePipe, fieldSelector );

    // parse the "road" output
    Pipe roadPipe = new Pipe( "road", tsvCheck );
    regex = "^\\s+Sequence\\:.*\\s+Year Constructed\\:\\s+(\\d+)\\s+Traffic Count\\:\\s+(\\d+)\\s+Traffic Index\\:\\s+(\\w.*\\w)\\s+Traffic Class\\:\\s+(\\w.*\\w)\\s+Traffic Date.*\\s+Paving Length\\:\\s+(\\d+)\\s+Paving Width\\:\\s+(\\d+)\\s+Paving Area\\:\\s+(\\d+)\\s+Surface Type\\:\\s+(\\w.*\\w)\\s+Surface Thickness.*\\s+Bike Lane\\:\\s+(\\w+)\\s+Bus Route\\:\\s+(\\w+)\\s+Truck Route\\:\\s+(\\w+)\\s+Remediation.*$";
    roadPipe = new Each( roadPipe, new Fields( "misc" ), new RegexFilter( regex ) );
    Fields roadFields = new Fields( "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route" );
    int[] roadGroups = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
    parser = new RegexParser( roadFields, regex, roadGroups );
    roadPipe = new Each( roadPipe, new Fields( "misc" ), parser, Fields.ALL );

    // join with road metadata
    Pipe metaRoadPipe = new Pipe( "meta_road" );
    roadPipe = new HashJoin( roadPipe, new Fields( "surface_type" ), metaRoadPipe, new Fields( "pavement_type" ), new InnerJoin() );
    roadPipe = new Rename( roadPipe, new Fields( "blurb" ), new Fields( "road_name" ) );

    // estimate albedo based on the road surface age and pavement type
    Fields albedoArguments = new Fields( "year_construct", "albedo_new", "albedo_worn" );
    roadPipe = new Each( roadPipe, albedoArguments, new AlbedoFunction( new Fields( "albedo" ), 2002 ), Fields.ALL );

    // generate road segments, with midpoint, y=mx+b, and road_geohash for each
    Fields segmentArguments = new Fields( "geo" );
    Fields segmentResults = new Fields( "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "lat_mid", "lng_mid" );
    roadPipe = new Each( roadPipe, segmentArguments, new RoadSegmentFunction( segmentResults ), Fields.ALL );

    geohashArguments = new Fields( "lat_mid", "lng_mid" );
    roadPipe = new Each( roadPipe, geohashArguments, new GeoHashFunction( new Fields( "road_geohash" ), 6 ), Fields.ALL );

    fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "road_geohash" );
    roadPipe = new Retain( roadPipe, fieldSelector );

    // join the tree and road pipes to estimate shade
    Pipe shadePipe = new Pipe( "shade", roadPipe );
    shadePipe = new CoGroup( shadePipe, new Fields( "road_geohash" ), treePipe, new Fields( "tree_geohash" ), new InnerJoin() );

    // calculate a rough estimate for distance from tree to road, then filter for "< ~1 block"
    Fields treeDistArguments = new Fields( "tree_lat", "tree_lng", "lat0", "lng0", "lat1", "lng1" );
    Fields tree_dist = new Fields( "tree_dist" );
    shadePipe = new Each( shadePipe, treeDistArguments, new TreeDistanceFunction( tree_dist ), Fields.ALL );

    ExpressionFilter distFilter = new ExpressionFilter( "tree_dist > 25.0", Double.class );
    shadePipe = new Each( shadePipe, tree_dist, distFilter );

    // checkpoint this (big) calculation too
    fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "lat1", "lng1", "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_dist", "tree_geohash" );
    shadePipe = new Retain( shadePipe, fieldSelector );
    shadePipe = new GroupBy( shadePipe, new Fields( "tree_name" ), new Fields( "tree_dist" ) );

    Checkpoint shadeCheck = new Checkpoint( "shade", shadePipe );

    // determine the geohash for GPS tracks log events
    Pipe logsPipe = new Pipe( "logs" );
    geohashArguments = new Fields( "lat", "lng" );
    logsPipe = new Each( logsPipe, geohashArguments, new GeoHashFunction( new Fields( "gps_geohash" ), 6 ), Fields.ALL );
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

        CONN_FACTORY.createConnection(BOOTSTRAP_SERVER_INFO, callback, remoteExceptionHandler);
    try
    {
      log.info("initial setup");
      final List<String> sourceNamesList = Arrays.asList(SOURCE1_NAME);
      final Checkpoint cp = Checkpoint.createOnlineConsumptionCheckpoint(0);
      BootstrapCheckpointHandler cpHandler = new BootstrapCheckpointHandler(sourceNamesList);
      cpHandler.createInitialBootstrapCheckpoint(cp, 0L);
      final DummyDatabusBootstrapConnectionStateMessage bstCallback =
          new DummyDatabusBootstrapConnectionStateMessage(log);

      log.info("process a normal startSCN which should establish the connection");
      sendStartScnHappyPath(conn, cp, bstCallback, SOURCE1_NAME, 100L, log);
      Assert.assertTrue(conn.isConnected());

      //wait for the response
      TestUtil.assertWithBackoff(new ConditionCheck()
      {
        @Override
        public boolean check()
        {
          return null != bstCallback.getCheckpoint();
        }
      }, "wait for /startSCN response", 100, log);

      log.info("verify /startSCN response");
      final Checkpoint startScnCp = bstCallback.getCheckpoint();
      Assert.assertNotNull(startScnCp);
      Assert.assertEquals(100L, startScnCp.getBootstrapStartScn().longValue());

      log.info("instrument the client pipeline so that we can intercept and delay the channelClosed message");
      final Semaphore passMessage = new Semaphore(1);
      final CountDownLatch closeSent = new CountDownLatch(1);
      passMessage.acquire();
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

        if(_config.getPredicatePushDown())
          writer = createEventWriter(request, clientFreeBufferSize, null, enc);
        else
          writer = createEventWriter(request, clientFreeBufferSize, keyFilter, enc);

        Checkpoint cp = new Checkpoint(checkpointString);


      DbusClientMode consumptionMode = cp.getConsumptionMode();

      LOG.info("Bootstrap request received: " +
          "fetchSize=" + clientFreeBufferSize +
          ", consumptionMode=" + consumptionMode +
          ", checkpoint=" + checkpointString +
          ", predicatePushDown=" + _config.getPredicatePushDown()
          );

      try
      {
        boolean phaseCompleted = false;
        switch (consumptionMode)
        {
        case BOOTSTRAP_SNAPSHOT:
          phaseCompleted = processor.streamSnapShotRows(new Checkpoint(
              checkpointString), writer);
          break;
        case BOOTSTRAP_CATCHUP:
          phaseCompleted = processor.streamCatchupRows(new Checkpoint(
              checkpointString), writer);
          break;
        default:
          if (null != bootstrapStatsCollector)
            bootstrapStatsCollector.registerErrBootstrap();
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

        RequestProcessingException
    {
          initBootstrapServerInfo();

        String ckptStr = request.getParams().getProperty(CHECKPOINT_PARAM);
        Checkpoint ckpt = null;

        if ( null != ckptStr)
        {
          ckpt = new Checkpoint(ckptStr);

          String bsServerInfo = ckpt.getBootstrapServerInfo();

          if ((null != bsServerInfo) && (null != _serverInfo))
          {
            ServerInfo expServerInfo = null;
            try
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

  {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor  processor =
        new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler,
                                                   null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

  {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor  processor =
        new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler,
                                                   null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

  {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor  processor =
        new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler,
                                                   null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

  {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor  processor =
        new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler,
                                                   null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
    HttpChunkTrailer httpChunkTrailer = new DefaultHttpChunkTrailer();

    processor.startResponse(httpResponse);
    processor.addChunk(httpChunk);
    processor.addTrailer(httpChunkTrailer);
    processor.finishResponse();
    processor.channelException(new Exception("dummy exception"));

    Assert.assertEquals("Error Handled", true, processor._errorHandled);
    Assert.assertEquals("Processor Response State", AbstractHttpResponseProcessorDecorator.ResponseStatus.CHUNKS_EXCEPTION,processor._responseStatus);
    Assert.assertEquals("Actor Queue Size", 1, queue.getMessages().size());
    Assert.assertEquals("Expected ConnectionStateMessage","TestConnectionStateMessage", queue.getMessages().get(0).getClass().getSimpleName());
    TestConnectionStateMessage gotMsg = (TestConnectionStateMessage)(queue.getMessages().get(0));
    Assert.assertEquals("Expected ConnectionStateMessage State", TestConnectionStateMessage.State.STARTSCN_RESPONSE_SUCCESS, gotMsg._state);
    Assert.assertEquals("StartSCN Response Id Check",new Long(5678912),cp.getBootstrapStartScn());
  }
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

  {
    // Both Exception and ChannelClose getting triggered
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor  processor =
        new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler,
                                                   null);
    processor.channelException(new Exception("dummy exception"));
View Full Code Here

Examples of com.linkedin.databus.core.Checkpoint

  {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    DefaultHttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapTargetScnHttpResponseProcessor  processor =
        new BootstrapTargetScnHttpResponseProcessor(null, queue, stateMsg, cp,
                                                    remoteExHandler, null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
    HttpChunkTrailer httpChunkTrailer = new DefaultHttpChunkTrailer();

    processor.startResponse(httpResponse);
    processor.addChunk(httpChunk);
    processor.addTrailer(httpChunkTrailer);
    processor.finishResponse();

    Assert.assertEquals("Error Handled", false, processor._errorHandled);
    Assert.assertEquals("Processor Response State", AbstractHttpResponseProcessorDecorator.ResponseStatus.CHUNKS_FINISHED,processor._responseStatus);
    Assert.assertEquals("Actor Queue Size", 1, queue.getMessages().size());
    Assert.assertEquals("Expected ConnectionStateMessage","TestConnectionStateMessage", queue.getMessages().get(0).getClass().getSimpleName());
    TestConnectionStateMessage gotMsg = (TestConnectionStateMessage)(queue.getMessages().get(0));
    Assert.assertEquals("Expected ConnectionStateMessage State", TestConnectionStateMessage.State.TARGETSCN_RESPONSE_SUCCESS, gotMsg._state);
    Assert.assertEquals("StartSCN Response Id Check",new Long(5678912),cp.getBootstrapTargetScn());
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.