Package com.facebook.hiveio.input

Examples of com.facebook.hiveio.input.HiveInputDescription

196
197
198
199
200
201
202
203
204
205
206
207
    createTestTable();

    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.setTableDesc(hiveTableDesc);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setTableDesc(hiveTableDesc);

    HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(),
        null, hiveTableDesc);

    List<HiveWritableRecord> writeRecords = Lists.newArrayList();
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
    LOG.info("Creating Hive client for Metastore at {}", metastoreHostPort);
    ThriftHiveMetastore.Iface client = HiveMetastores.create(
        metastoreHostPort.host, metastoreHostPort.port);

    HiveInputDescription inputDesc = initInput(metastoreHostPort);

    HiveStats hiveStats = HiveUtils.statsOf(client, inputDesc);
    LOG.info("{}", hiveStats);

    HiveConf hiveConf = HiveUtils.newHiveConf(TailerCmd.class);
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
   *
   * @param metastoreHostPort metastore location info
   * @return HiveInputDescription
   */
  private HiveInputDescription initInput(HostPort metastoreHostPort) {
    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.getTableDesc().setDatabaseName(args.inputTable.database);
    inputDesc.getTableDesc().setTableName(args.inputTable.table);
    inputDesc.setPartitionFilter(args.inputTable.partitionFilter);
    args.splits.compute(args.multiThread.threads);
    inputDesc.setNumSplits(args.splits.requestNumSplits);
    inputDesc.getMetastoreDesc().setHost(metastoreHostPort.host);
    inputDesc.getMetastoreDesc().setPort(metastoreHostPort.port);
    return inputDesc;
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
    InternalVertexRunner.run(conf, new String[0], new String[0]);

    Helpers.commitJob(conf);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.getTableDesc().setTableName(outputTable);

    Iterator<HiveReadableRecord> records = HiveInput.readTable(inputDesc).iterator();

    int expected[] = { -1, 1, 2, -1, 1 };
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
  }

  @Test
  public void testCheckEdge() throws Exception {
    HiveToEdge hiveToEdge = new HiveIntNullEdge();
    HiveInputDescription inputDesc = new HiveInputDescription();
    HiveTableSchema schema = TestSchema.builder()
        .addColumn("foo", HiveType.INT)
        .addColumn("bar", HiveType.INT)
        .build();
    hiveToEdge.checkInput(inputDesc, schema);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
  }

  @Test
  public void testCheckVertex() throws Exception {
    HiveToVertex hiveToVertex = new HiveIntNullNullVertex();
    HiveInputDescription inputDesc = new HiveInputDescription();
    HiveTableSchema schema = TestSchema.builder()
        .addColumn("foo", HiveType.INT)
        .addColumn("bar", HiveType.LIST)
        .build();
    hiveToVertex.checkInput(inputDesc, schema);
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
    InternalVertexRunner.run(conf, new String[0], new String[0]);

    Helpers.commitJob(conf);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.getTableDesc().setTableName(outputTable);

    Iterator<HiveReadableRecord> records = HiveInput.readTable(inputDesc).iterator();

    printRecords(HiveInput.readTable(inputDesc));
View Full Code Here
62
63
64
65
66
67
68
69
70
71
       " (i1 BIGINT, i2 BIGINT) ");

    GiraphConfiguration conf = new GiraphConfiguration();
    runJob(tableName, conf);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.getTableDesc().setTableName(tableName);

    verifyRecords(inputDesc);
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
    GiraphConfiguration conf = new GiraphConfiguration();
    GiraphHiveConstants.HIVE_VERTEX_OUTPUT_PARTITION.set(conf, "ds=foobar");

    runJob(tableName, conf);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.getTableDesc().setTableName(tableName);
    inputDesc.setPartitionFilter("ds='foobar'");

    verifyRecords(inputDesc);
  }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
    conf.setVertexOutputFormatThreadSafe(true);
    conf.setNumOutputThreads(2);
    GiraphConstants.USER_PARTITION_COUNT.set(conf, 4);
    runJob(tableName, conf);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.getTableDesc().setTableName(tableName);

    verifyRecords(inputDesc);
  }
View Full Code Here
TOP

Related Classes of com.facebook.hiveio.input.HiveInputDescription

  • com.facebook.hiveio.benchmark.InputBenchmark
  • com.facebook.hiveio.common.HiveTableName
  • com.facebook.hiveio.output.OutputTest
  • com.facebook.hiveio.output.TypeUpgradeTest
  • com.facebook.hiveio.tailer.Tailer
  • com.facebook.hiveio.tailer.TailerCmd
  • org.apache.giraph.hive.common.HiveInputOptions
  • org.apache.giraph.hive.input.CheckInputTest
  • org.apache.giraph.hive.input.edge.HiveEdgeInputFormat
  • org.apache.giraph.hive.input.mapping.HiveMappingInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.