Package com.facebook.hiveio.input

Examples of com.facebook.hiveio.input.HiveInputDescription

55
56
57
58
59
60
61
62
63
64
65
66
67
68
    HadoopNative.requireHadoopNative();

    Timer allTime = Metrics.newTimer(InputBenchmark.class, "all-time", MILLISECONDS, MILLISECONDS);
    TimerContext allTimerContext = allTime.time();

    HiveInputDescription input = new HiveInputDescription();
    input.setDbName(args.tableOpts.database);
    input.setTableName(args.tableOpts.table);
    input.setPartitionFilter(args.tableOpts.partitionFilter);

    HiveConf hiveConf = new HiveConf(InputBenchmark.class);
    ThriftHiveMetastore.Iface client = HiveMetastores.create(args.metastoreOpts.hiveHost, args.metastoreOpts.hivePort);

    System.err.println("Initialize profile with input data");
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
    LOG.info("Creating Hive client for Metastore at {}", metastoreHostPort);
    ThriftHiveMetastore.Iface client = HiveMetastores.create(metastoreHostPort.host,
        metastoreHostPort.port);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setDbName(opts.tableOpts.database);
    inputDesc.setTableName(opts.tableOpts.table);
    inputDesc.setPartitionFilter(opts.tableOpts.partitionFilter);
    if (opts.requestNumSplits == 0) {
      opts.requestNumSplits = opts.threads * opts.requestSplitsPerThread;
    }
    inputDesc.setNumSplits(opts.requestNumSplits);

    HiveStats hiveStats = HiveUtils.statsOf(client, inputDesc);
    LOG.info("{}", hiveStats);

    HiveConf hiveConf = new HiveConf(Tailer.class);
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    HadoopNative.requireHadoopNative();

    Timer allTime = Metrics.newTimer(InputBenchmark.class, "all-time", MILLISECONDS, MILLISECONDS);
    TimerContext allTimerContext = allTime.time();

    HiveInputDescription input = new HiveInputDescription();
    input.setDbName(args.tableOpts.database);
    input.setTableName(args.tableOpts.table);
    input.setPartitionFilter(args.tableOpts.partitionFilter);
    input.getMetastoreDesc().setHost(args.metastoreOpts.host);
    input.getMetastoreDesc().setPort(args.metastoreOpts.port);

    HiveConf hiveConf = HiveUtils.newHiveConf(InputBenchmark.class);

    System.err.println("Initialize profile with input data");
    HiveApiInputFormat.setProfileInputDesc(hiveConf, input, DEFAULT_PROFILE_ID);
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    LOG.info("Creating Hive client for Metastore at {}", metastoreHostPort);
    ThriftHiveMetastore.Iface client = HiveMetastores.create(
        metastoreHostPort.host, metastoreHostPort.port);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setDbName(args.inputTable.database);
    inputDesc.setTableName(args.inputTable.table);
    inputDesc.setPartitionFilter(args.inputTable.partitionFilter);
    if (args.requestNumSplits == 0) {
      args.requestNumSplits = args.multiThread.threads * args.requestSplitsPerThread;
    }
    inputDesc.setNumSplits(args.requestNumSplits);
    inputDesc.getMetastoreDesc().setHost(metastoreHostPort.host);
    inputDesc.getMetastoreDesc().setPort(metastoreHostPort.port);

    HiveStats hiveStats = HiveUtils.statsOf(client, inputDesc);
    LOG.info("{}", hiveStats);

    HiveConf hiveConf = HiveUtils.newHiveConf(TailerCmd.class);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
    HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(),
        null, hiveTableDesc);

    writeData(outputDesc, schema);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setTableDesc(hiveTableDesc);

    verifyData(inputDesc);
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(),
        null, hiveTableDesc);

    writeData(outputDesc, schema);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setPartitionFilter("ds='foobar'");
    inputDesc.setTableDesc(hiveTableDesc);

    verifyData(inputDesc);
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
    createTestTable();

    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.setTableDesc(hiveTableDesc);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setTableDesc(hiveTableDesc);

    HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(),
        null, hiveTableDesc);

    List<HiveWritableRecord> writeRecords = Lists.newArrayList();
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
    createTestTable();

    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.setTableDesc(hiveTableDesc);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setTableDesc(hiveTableDesc);

    HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(),
        null, hiveTableDesc);

    List<HiveWritableRecord> writeRecords = Lists.newArrayList();
View Full Code Here
66
67
68
69
70
71
72
73
74
75
    HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(),
        null, hiveTableDesc);

    writeData(outputDesc, schema);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setTableDesc(hiveTableDesc);

    verifyData(inputDesc);
  }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
    HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(),
        null, hiveTableDesc);

    writeData(outputDesc, schema);

    HiveInputDescription inputDesc = new HiveInputDescription();
    inputDesc.setPartitionFilter("ds='foobar'");
    inputDesc.setTableDesc(hiveTableDesc);

    verifyData(inputDesc);
  }
View Full Code Here
TOP

Related Classes of com.facebook.hiveio.input.HiveInputDescription

  • com.facebook.hiveio.benchmark.InputBenchmark
  • com.facebook.hiveio.common.HiveTableName
  • com.facebook.hiveio.output.OutputTest
  • com.facebook.hiveio.output.TypeUpgradeTest
  • com.facebook.hiveio.tailer.Tailer
  • com.facebook.hiveio.tailer.TailerCmd
  • org.apache.giraph.hive.common.HiveInputOptions
  • org.apache.giraph.hive.input.CheckInputTest
  • org.apache.giraph.hive.input.edge.HiveEdgeInputFormat
  • org.apache.giraph.hive.input.mapping.HiveMappingInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.