Package cascading.flow.hadoop

Examples of cascading.flow.hadoop.HadoopFlowProcess

105
106
107
108
109
110
111
112
113
114
115
        INPUT_PATH);
    Hfs outputTap = new Hfs(new ProtobufScheme("leader.name", Example.Partnership.class),
        OUTPUT_PATH);

    // make sure the input path exists
    inputTap.openForWrite(new HadoopFlowProcess()).close();
    // make sure the output path does not exist
    FileSystem.get(new Configuration()).delete(new Path(OUTPUT_PATH), true);

    FlowDef flowDef = new FlowDef()
        .addSource("input", inputTap)
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
            new ExtractProto(Example.Partnership.class, "leader").getEmittedClasses());
  }

  private static Tuple exec(Function f, final Tuple input) {
    final AtomicReference<Tuple> output = new AtomicReference<Tuple>();
    f.prepare(new HadoopFlowProcess(), null);

    f.operate(new HadoopFlowProcess(), new FunctionCall() {
      @Override public TupleEntry getArguments() {
        return new TupleEntry(new Fields("blah"), input);
      }

      @Override public Fields getDeclaredFields() {
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
  }

  public void testWrongArgumentClass() throws Exception {
    AbstractExpandProto func = new ExpandProto(Example.Person.class, "name");
    try {
      func.operate(new HadoopFlowProcess(), new FunctionCall() {
        @Override public TupleEntry getArguments() {
          return new TupleEntry(new Fields("partnership"), new Tuple(Example.Partnership.newBuilder().setFollower(
              Example.Person.newBuilder().setName("bryan")).setLeader(Example.Person.newBuilder().setName("alsoBryan")).build()));
        }
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
  public void testInFlow() throws Exception {
    FileSystem.get(new Configuration()).delete(new Path("/tmp/input"), true);
    FileSystem.get(new Configuration()).delete(new Path("/tmp/output"), true);

    Hfs inTap = new Hfs(new ProtobufScheme("value", Example.Person.class), "/tmp/input");
    TupleEntryCollector collector = inTap.openForWrite(new HadoopFlowProcess());
    collector.add(new TupleEntry(new Fields("value"), new Tuple(BRYAN.build())));
    collector.add(new TupleEntry(new Fields("value"), new Tuple(LUCAS.build())));
    collector.close();

    Pipe inPipe = new Pipe("in");
    Pipe p = new Each(inPipe, new Fields("value"), new ExpandProto(Example.Person.class), new Fields("id", "name", "email", "position"));

    Hfs sink = new Hfs(new TextLine(), "/tmp/output");
    new HadoopFlowConnector().connect(inTap, sink, p).complete();

    TupleEntryIterator iter = sink.openForRead(new HadoopFlowProcess());
    List<Tuple> results = new ArrayList<Tuple>();
    while (iter.hasNext()) {
      results.add(iter.next().getTupleCopy());
    }
    assertEquals(2, results.size());
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
        return temp.getAbsoluteFile().getPath();
    }

    @Override
    public boolean commitResource(JobConf conf) throws java.io.IOException {
        TupleEntryIterator it = new HadoopFlowProcess(conf).openTapForRead(this);
        System.out.println("");
        System.out.println("");
        System.out.println("RESULTS");
        System.out.println("-----------------------");
        while (it.hasNext()) {
View Full Code Here
90
91
92
93
94
95
96
    }

    @Override
    public TupleEntryIterator openForRead(FlowProcess<Properties> flowProcess, RecordReader input) throws IOException {
        JobConf jobConf = mergeDefaults("LocalTap#openForRead", flowProcess.getConfigCopy(), defaults);
        return lfs.openForRead(new HadoopFlowProcess(jobConf));
    }
View Full Code Here
97
98
99
100
101
102
103
    @Override
    public TupleEntryCollector openForWrite(FlowProcess<Properties> flowProcess, OutputCollector output)
            throws IOException {
        JobConf jobConf = mergeDefaults("LocalTap#openForWrite", flowProcess.getConfigCopy(), defaults);
        return lfs.openForWrite(new HadoopFlowProcess(jobConf));
    }
View Full Code Here
159
160
161
162
163
164
165
        }

        @Override
        public Fields retrieveSourceFields(FlowProcess<Properties> flowProcess,
                Tap tap) {
            return scheme.retrieveSourceFields(new HadoopFlowProcess(defaults), lfs);
        }
View Full Code Here
165
166
167
168
169
170
171
        }

        @Override
        public void presentSourceFields(FlowProcess<Properties> flowProcess,
                Tap tap, Fields fields) {
            scheme.presentSourceFields(new HadoopFlowProcess(defaults), lfs, fields);
        }
View Full Code Here
172
173
174
175
176
177
178
179
        @Override
        public void sourceConfInit(FlowProcess<Properties> flowProcess,
                Tap<Properties, RecordReader, OutputCollector> tap, Properties conf) {
            JobConf jobConf = mergeDefaults("LocalScheme#sourceConfInit", conf, defaults);
            scheme.sourceConfInit(new HadoopFlowProcess(jobConf), lfs, jobConf);
            overwriteProperties(conf, jobConf);
        }
View Full Code Here
TOP

Related Classes of cascading.flow.hadoop.HadoopFlowProcess

  • cascading.flow.hadoop.stream.graph.HadoopMapStreamGraph
  • cascading.flow.hadoop.util.HadoopMRUtil
  • cascading.lingual.platform.hadoop.HadoopPlatformBroker
  • cascading.lingual.platform.hadoop2.Hadoop2MR1PlatformBroker
  • cascading.platform.hadoop.HadoopPlatform
  • cascading.platform.hadoop2.Hadoop2MR1Platform
  • cascading.tuple.hadoop.SpillableTupleHadoopTest
  • cascading.tuple.TupleEntryCollector
  • com.etsy.cascading.tap.local.LocalTap
  • com.etsy.cascading.tap.local.LocalTap$LocalScheme
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.