Package com.datasalt.pangool.io

Examples of com.datasalt.pangool.io.Schema

84
85
86
87
88
89
90
  public static Schema getOutputCountSchema() {
    List<Field> fields = new ArrayList<Field>();
    fields.add(Field.create("topic", Type.INT));
    fields.add(Field.create("totalcount", Type.INT));
    return new Schema("outputcount", fields);
  }
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
    List<Field> fields = new ArrayList<Field>();
    fields.add(Field.create("location", Type.STRING));
    fields.add(Field.create("date", Type.STRING));
    fields.add(Field.create("hashtag", Type.STRING));
    fields.add(Field.create("count", Type.INT));
    Schema schema = new Schema("my_schema", fields);

    TupleMRBuilder mr = new TupleMRBuilder(conf);
    mr.addIntermediateSchema(schema);
    mr.setGroupByFields("location", "date", "hashtag");
    mr.setOrderBy(new OrderBy().add("location", Order.ASC).add("date", Order.ASC).add("hashtag", Order.ASC));
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
    fields.add(Field.create("user", Type.STRING));
    fields.add(Field.create("feature", Type.STRING));
    fields.add(Field.create("all",Type.BOOLEAN));
    fields.add(Field.create("clicks", Type.INT));

    Schema schema = new Schema("my_schema", fields);

    TupleMRBuilder mr = new TupleMRBuilder(conf);
    mr.addIntermediateSchema(schema);
    mr.setGroupByFields("user", "all", "feature");
    mr.setOrderBy(new OrderBy().add("user", Order.ASC).add("all", Order.DESC).add("feature", Order.ASC));
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
    List<Field> fields = new ArrayList<Field>();
    fields.add(Field.create("intField", Type.INT));
    fields.add(Field.create("strField", Type.STRING));
    fields.add(Field.create("longField", Type.LONG));
    fields.add(Field.create("doubleField", Type.DOUBLE));
    Schema schema = new Schema("schema", fields);

    TupleMRBuilder mr = new TupleMRBuilder(conf, "Pangool Secondary Sort");
    mr.addIntermediateSchema(schema);
    mr.setGroupByFields("intField", "strField");
    mr.setOrderBy(new OrderBy().add("intField", Order.ASC).add("strField", Order.ASC).add("longField", Order.ASC));
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
    List<Field> urlMapFields = new ArrayList<Field>();
    urlMapFields.add(Field.create("url",Type.STRING));
    urlMapFields.add(Field.create("canonicalUrl",Type.STRING));

    TupleMRBuilder mr = new TupleMRBuilder(conf,"Pangool Url Resolution");
    mr.addIntermediateSchema(new Schema("urlMap", urlMapFields));
    mr.addIntermediateSchema(new Schema("urlRegister", urlRegisterFields));
    mr.setGroupByFields("url");
    mr.setTupleReducer(new Handler());
    mr.setOutput(new Path(output), new HadoopOutputFormat(TextOutputFormat.class), Text.class, NullWritable.class);
    mr.addInput(new Path(input1), new HadoopInputFormat(TextInputFormat.class), new UrlMapProcessor());
    mr.addInput(new Path(input2), new HadoopInputFormat(TextInputFormat.class), new UrlProcessor());
View Full Code Here
108
109
110
111
112
113
114
    List<Field> fields = new ArrayList<Field>();
    // The schema has 3 fields: word, topicId and count
    fields.add(Field.create("word", Type.STRING));
    fields.add(Field.create("topic", Type.INT));
    fields.add(Field.create("count", Type.INT));
    return new Schema("schema", fields);
  }
View Full Code Here
52
53
54
55
56
57
58
    // Configure schema, sort and group by
    List<Field> fields = new ArrayList<Field>();
    fields.add(Field.create("first", Type.INT));
    fields.add(Field.create("second", Type.INT));

    return new Schema("my_schema", fields);
  }
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
    fields.add(Field.create("user", Type.STRING));
    fields.add(Field.create("feature", Type.STRING));
    fields.add(Field.create("all", Type.BOOLEAN));
    fields.add(Field.create("clicks", Type.INT));

    Schema schema = new Schema("my_schema", fields);

    TupleMRBuilder mr = new TupleMRBuilder(conf);
    mr.addIntermediateSchema(schema);
    mr.setGroupByFields("user", "all", "feature");
    mr.setOrderBy(new OrderBy().add("user", Order.ASC).add("all", Order.DESC).add("feature", Order.ASC));
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
    List<Field> fields = new ArrayList<Field>();
    fields.add(Field.create("location", Type.STRING));
    fields.add(Field.create("date", Type.STRING));
    fields.add(Field.create("hashtag", Type.STRING));
    fields.add(Field.create("count", Type.INT));
    Schema schema = new Schema("my_schema", fields);

    TupleMRBuilder mr = new TupleMRBuilder(conf);
    mr.addIntermediateSchema(schema);
    mr.setGroupByFields("location", "date", "hashtag");
    mr.setOrderBy(new OrderBy().add("location", Order.ASC).add("date", Order.ASC)
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
    final int iterations = conf.getInt("gol.iterations", 1000);
    Log.info("using parameters: maxX grid: " + maxX + " maxY grid: " + maxY + " max #iterations: "
        + iterations);

    // Define the intermediate schema: a pair of ints
    final Schema schema = new Schema("minMax", Fields.parse("min:int, max:int"));

    TupleMRBuilder job = new TupleMRBuilder(conf);
    job.addIntermediateSchema(schema);
    job.setGroupByFields("min", "max");
    job.setCustomPartitionFields("min");
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.Schema

  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin
  • com.datasalt.pangool.examples.gameoflife.GameOfLifeJob
  • com.datasalt.pangool.examples.movingaverage.MovingAverage
  • com.datasalt.pangool.examples.secondarysort.SecondarySort
  • com.datasalt.pangool.examples.simplesecondarysort.SimpleSecondarySort
  • com.datasalt.pangool.examples.topicalwordcount.TopicalWordCount
  • com.datasalt.pangool.examples.topicalwordcount.TopicFingerprint
  • com.datasalt.pangool.examples.topnhashtags.TopNHashTags
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.