Examples of SequenceFile

cascading.scheme.SequenceFile
cascading.scheme.hadoop.SequenceFile
A SequenceFile is a type of {@link cascading.scheme.Scheme}, which is a flat file consisting of binary key/value pairs. This is a space and time efficient means to store data.

Examples of cascading.scheme.hadoop.SequenceFile

  private final String INPUT_PATH = getTestRoot() + "/input";
  private final String OUTPUT_PATH = getTestRoot() + "/output";


  @Test
  public void testMain() throws IOException {
    Hfs source = new Hfs(new SequenceFile(new Fields("key")), INPUT_PATH);
    TupleEntryCollector tc = source.openForWrite(CascadingUtil.get().getFlowProcess());
    tc.add(new Tuple("k0"));
    tc.add(new Tuple("k0"));
    tc.add(new Tuple("k1"));
    tc.add(new Tuple("k1"));
    tc.add(new Tuple("k1"));
    tc.add(new Tuple("k2"));
    tc.add(new Tuple("k2"));
    tc.close();


    Tap sink = new Hfs(new SequenceFile(new Fields("key", "count")), OUTPUT_PATH);
    
    Pipe pipe = new Pipe("pipe");
    pipe = Combiner.assembly(pipe, new CountExactAggregator(), new Fields("key"), new Fields(), new Fields("count"));


    CascadingUtil.get().getFlowConnector().connect(source, sink, pipe).complete();

View Full Code Here

Examples of cascading.scheme.hadoop.SequenceFile


  @Before
  public void bloomAssemblySetUp() throws Exception {
    CascadingUtil.get().setDefaultProperty(BloomProps.KEY_SAMPLE_RATE, 1.0);


    lhsStore = new Hfs(new SequenceFile(new Fields("key", "key2", "lhs-value")), getTestRoot() + "/lhs");
    lhs2Store = new Hfs(new SequenceFile(new Fields("key", "key2", "lhs-value")), getTestRoot() + "/lhs2");
    rhsStore = new Hfs(new SequenceFile(new Fields("key", "key2", "rhs-value")), getTestRoot() + "/rhs");
    rhs2Store = new Hfs(new SequenceFile(new Fields("key", "key2", "rhs-value")), getTestRoot() + "/rhs2");


    TapHelper.writeToTap(rhsStore,
        new Tuple(bytes("1"), bytes("11"), "a-rhs"),
        new Tuple(bytes("1"), bytes("11"), "b-rhs"),
        new Tuple(bytes("2"), bytes("12"), "c-rhs"),

View Full Code Here

Examples of cascading.scheme.hadoop.SequenceFile

  protected Hfs output3;


  @Before
  public void setUp() throws Exception {


    output = new Hfs(new SequenceFile(new Fields("lhs-key", "lhs-key2", "lhs-value", "rhs-key", "rhs-key2", "rhs-value")),
        getTestRoot() + "/output1");


    output2 = new Hfs(new SequenceFile(new Fields("lhs-key-renamed", "lhs-value-renamed", "lhs-key", "rhs-value")),
        getTestRoot() + "/output2");


    output3 = new Hfs(new SequenceFile(new Fields("key", "key2", "v1", "key3", "key4", "v2")),
        getTestRoot() + "/output3");
  }

View Full Code Here

Examples of cascading.scheme.hadoop.SequenceFile

public class TestRenameInPlace extends BaseTestCase {


  @Test
  public void testIt() throws IOException {


    Hfs source1 = new Hfs(new SequenceFile(new Fields("key", "num")), getTestRoot()+"/input");
    TapHelper.writeToTap(source1,
        new Tuple(3, 4));


    Hfs sink = new Hfs(new SequenceFile(new Fields("key2", "num")), getTestRoot()+"/output");


    Pipe input = new Pipe("input");
    input = new RenameInPlace(input, new Fields("key", "num"), new Fields("key"), new Fields("key2"));

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.