Examples of cascading.flow.hadoop.HadoopFlowProcess

cascading.flow.hadoop.HadoopFlowProcess
Class HadoopFlowProcess is an implementation of {@link FlowProcess} for Hadoop. Use this interface to get directaccess to the Hadoop JobConf and Reporter interfaces.
Be warned that coupling to this implementation will cause custom {@link cascading.operation.Operation}s to fail if they are executed on a system other than Hadoop. @see cascading.flow.FlowSession @see JobConf @see Reporter

        }


        @Override
        public Fields retrieveSinkFields(FlowProcess<Properties> flowProcess,
                Tap tap) {
            return scheme.retrieveSinkFields(new HadoopFlowProcess(defaults), lfs);
        }

View Full Code Here

        }


        @Override
        public void presentSinkFields(FlowProcess<Properties> flowProcess, 
                Tap tap, Fields fields) {
            scheme.presentSinkFields(new HadoopFlowProcess(defaults), lfs, fields);
        }

View Full Code Here

            
        @Override
        public void sinkConfInit(FlowProcess<Properties> flowProcess,
                Tap<Properties, RecordReader, OutputCollector> tap, Properties conf) {
            JobConf jobConf = mergeDefaults("LocalScheme#sinkConfInit", conf, defaults);
            scheme.sinkConfInit(new HadoopFlowProcess(jobConf), lfs, jobConf);
            overwriteProperties(conf, jobConf);
        }

View Full Code Here

        return temp.getAbsoluteFile().getPath();
    }


    @Override
    public boolean commitResource(JobConf conf) throws java.io.IOException {
        TupleEntryIterator it = new HadoopFlowProcess(conf).openTapForRead(this);


        boolean first_time = true;
        while (it.hasNext()) {
            TupleEntry tuple = it.next();
            results.add(tuple.getTupleCopy());

View Full Code Here


      try {
        LOG.info("HLL counter found " + approxCounter.cardinality() + " distinct keys");


        Hfs tap = new Hfs(new SequenceFile(new Fields("bytes")), BloomProps.getApproxCountsDir(conf));
        TupleEntryCollector out = tap.openForWrite(new HadoopFlowProcess(conf));
        out.add(new Tuple(new BytesWritable(approxCounter.getBytes())));
        out.close();


      } catch (IOException e) {
        throw new RuntimeException("couldn't write approximate counts to side bucket", e);

View Full Code Here

  public FlowProcess<JobConf> getFlowProcess() {
    return getFlowProcess(getJobConf());
  }


  public FlowProcess<JobConf> getFlowProcess(JobConf jobConf) {
    return new HadoopFlowProcess(jobConf);
  }

View Full Code Here

      maxHashes = BloomProps.getMaxBloomHashes(conf);
      minHashes = BloomProps.getMinBloomHashes(conf);


      for (int i = minHashes; i <= maxHashes; i++) {
        Hfs tap = new Hfs(new SequenceFile(new Fields("split", "filter")), partsRoot + "/" + i + "/");
        numHashesToCollector.put(i, tap.openForWrite(new HadoopFlowProcess(conf)));
      }


    } catch (IOException e) {
      throw new RuntimeException(e);
    }

View Full Code Here

    }


  @Override
  public FlowProcess<JobConf> getFlowProcess()
    {
    return new HadoopFlowProcess( getPlannerConfig() );
    }

View Full Code Here

    }


  @Override
  public FlowProcess<JobConf> getFlowProcess()
    {
    return new HadoopFlowProcess( getPlannerConfig() );
    }

View Full Code Here


  @Test
  public void testHadoopConf() {
    CombinedSequenceFile csfScheme = new CombinedSequenceFile(Fields.ALL);
    JobConf conf = new JobConf();
    FlowProcess fp = new HadoopFlowProcess();
    Tap<JobConf, RecordReader, OutputCollector> tap =
        new TempHfs(conf, "test", CombinedSequenceFile.class, false);


    csfScheme.sourceConfInit(fp, tap, conf);

View Full Code Here

0 1 2

TOP

Related Classes of cascading.flow.hadoop.HadoopFlowProcess

cascading.flow.hadoop.stream.graph.HadoopMapStreamGraph

cascading.flow.hadoop.util.HadoopMRUtil

cascading.lingual.platform.hadoop.HadoopPlatformBroker

cascading.lingual.platform.hadoop2.Hadoop2MR1PlatformBroker

cascading.platform.hadoop.HadoopPlatform

cascading.platform.hadoop2.Hadoop2MR1Platform

cascading.tuple.hadoop.SpillableTupleHadoopTest

cascading.tuple.TupleEntryCollector

com.etsy.cascading.tap.local.LocalTap

com.etsy.cascading.tap.local.LocalTap$LocalScheme

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.