Package cascading.pipe

Examples of cascading.pipe.Pipe

74
75
76
77
78
79
80
81
82
83
84

    @Test
    public void testReadFromES() throws Exception {
        Tap in = new EsTap(indexPrefix + "cascading-hadoop/artists");
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, new FilterNotNull());
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(5));
        // can't select when using unknown
        //pipe = new Each(pipe, new Fields("name"), AssertionLevel.STRICT, new AssertNotNull());
        pipe = new GroupBy(pipe);
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
        Properties cfg = cfg();
        cfg.setProperty("es.mapping.names", "nested:data.map.key");

        Tap in = new EsTap(indexPrefix + "cascading-hadoop/nestedmap", new Fields("nested"));
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, new FilterNotNull());
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(2));

        // print out
        Tap out = new HadoopPrintStreamTap(Stream.NULL);
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
    }

    @Test
    public void testReadFromES() throws Exception {
        Tap in = new EsTap("cascading-hadoop/artists", query);
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(5));
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertNotNull());

        // print out
        Tap out = new HadoopPrintStreamTap(Stream.NULL);
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94

    @Test
    public void testReadFromESWithFields() throws Exception {
        Tap in = new EsTap("cascading-hadoop/artists", query, new Fields("url", "name"));
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeEquals(2));
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertNotNull());

        // print out
        Tap out = new HadoopPrintStreamTap(Stream.NULL);
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
    }

    @Test
    public void testReadFromESAliasedField() throws Exception {
        Tap in = new EsTap("cascading-hadoop/alias", query, new Fields("address"));
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertNotNull());

        // print out
        Tap out = new HadoopPrintStreamTap(Stream.NULL);
        build(cfg(), in, out, pipe);
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
    }

    @Test
    public void testReadFromESWithFieldAlias() throws Exception {
        Tap in = new EsTap("cascading-hadoop/alias", query, new Fields("url"));
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertNotNull());

        // print out
        Tap out = new HadoopPrintStreamTap(Stream.NULL);
        Properties cfg = cfg();
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
        Properties cfg = cfg();
        cfg.setProperty("es.mapping.names", "nested:data.map.key");

        Tap in = new EsTap("cascading-hadoop/nestedmap", new Fields("nested"));
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, new FilterNotNull());
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(2));

        // print out
        Tap out = new HadoopPrintStreamTap(Stream.NULL);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85

    @Test
    public void testReadFromES() throws Exception {
        Tap in = new EsTap(indexPrefix + "cascading-local/artists");
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, new FilterNotNull());
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(5));
        // can't select when using unknown
        //pipe = new Each(pipe, new Fields("name"), AssertionLevel.STRICT, new AssertNotNull());
        pipe = new GroupBy(pipe);
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
        Properties cfg = cfg();
        cfg.setProperty("es.mapping.names", "nested:data.map.key");

        Tap in = new EsTap(indexPrefix + "cascading-local/nestedmap", new Fields("nested"));
        Pipe pipe = new Pipe("copy");
        pipe = new Each(pipe, new FilterNotNull());
        pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(2));

        // print out
        Tap out = new OutputStreamTap(new TextLine(), OUT);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    public void importUrls(boolean debug) throws Exception {


        try {
            Tap urlSource = _platform.makeTap(_platform.makeTextScheme(), _inputFilePath);
            Pipe importPipe = new Each("url importer", new Fields("line"), new CreateUrlFromTextFunction(new SimpleUrlNormalizer(), new SimpleUrlValidator()));

            Tap urlSink = _platform.makeTap(_platform.makeBinaryScheme(CrawlDbDatum.FIELDS), _destDirPath, SinkMode.REPLACE);

            FlowConnector flowConnector = _platform.makeFlowConnector();
            Flow flow = flowConnector.connect(urlSource, urlSink, importPipe);
View Full Code Here
TOP

Related Classes of cascading.pipe.Pipe

  • bixo.fetcher.FetcherTest
  • bixo.pipes.AbstractFetchPipeTest
  • cascading.BasicPipesPlatformTest
  • cascading.BasicTrapPlatformTest
  • cascading.BufferPipesPlatformTest
  • cascading.cascade.CascadePlatformTest
  • cascading.cascade.hadoop.RiffleCascadePlatformTest
  • cascading.cascade.ParallelCascadePlatformTest
  • cascading.CoGroupFieldedPipesPlatformTest
  • cascading.DistanceUseCasePlatformTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.