Package com.facebook.hiveio.schema

Examples of com.facebook.hiveio.schema.HiveTableSchema

84
85
86
87
88
89
90
91
92
93
94
    HiveOutputDescription outputDesc = new HiveOutputDescription();
    outputDesc.putPartitionValue("ds", "foobar");
    outputDesc.setTableDesc(hiveTableDesc);

    HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(),
        null, hiveTableDesc);

    writeData(outputDesc, schema);

    HiveInputDescription inputDesc = new HiveInputDescription();
View Full Code Here
59
60
61
62
63
64
65
66
67
  }

  @Override public void checkInputSpecs(Configuration conf) {
    HiveInputDescription inputDesc =
        GiraphHiveConstants.HIVE_VERTEX_INPUT.makeInputDescription(conf);
    HiveTableSchema schema = getTableSchema();
    HiveToEdge<I, E> hiveToEdge = HiveUtils.newHiveToEdge(getConf(), schema);
    hiveToEdge.checkInput(inputDesc, schema);
  }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
  @Override
  public void checkOutputSpecs(JobContext context)
    throws IOException, InterruptedException {
    hiveOutputFormat.checkOutputSpecs(context);
    HiveTableSchema schema = hiveOutputFormat.getTableSchema(getConf());
    VertexToHive<I, V, E> vertexToHive = newVertexToHive(getConf(), schema);
    vertexToHive.checkOutput(makeOutputDesc(), schema,
        HiveRecordFactory.newWritableRecord(schema));
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
  @Override
  public void checkInputSpecs(Configuration conf) {
    HiveInputDescription inputDesc =
        GiraphHiveConstants.HIVE_VERTEX_INPUT.makeInputDescription(conf);
    HiveTableSchema schema = getTableSchema();
    HiveToVertex<I, V, E> hiveToVertex = newHiveToVertex(getConf(), schema);
    hiveToVertex.checkInput(inputDesc, schema);
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
  public void checkOutput(HiveOutputDescription outputDesc,
      HiveTableSchema schema, HiveWritableRecord emptyRecord) { }

  @Override
  public void initialize() {
    HiveTableSchema schema = getTableSchema();
    ImmutableClassesGiraphConfiguration conf = getConf();

    vertexIdWriter = HiveJythonUtils.newValueWriter(schema,
        VERTEX_ID_COLUMN, conf, GraphType.VERTEX_ID,
        GiraphHiveConstants.VERTEX_ID_WRITER_JYTHON_NAME);
View Full Code Here
63
64
65
66
67
68
69
70
71
  @Override
  public void checkInputSpecs(Configuration conf) {
    HiveInputDescription inputDesc =
        GiraphHiveConstants.HIVE_MAPPING_INPUT.makeInputDescription(conf);
    HiveTableSchema schema = getTableSchema();
    HiveToMapping<I, B> hiveToMapping = newHiveToMapping(getConf(), schema);
    hiveToMapping.checkInput(inputDesc, schema);
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
  @Override
  public void initializeRecords(Iterator<HiveReadableRecord> records) {
    super.initializeRecords(records);

    HiveTableSchema schema = getTableSchema();
    ImmutableClassesGiraphConfiguration conf = getConf();

    sourceIdReader = HiveJythonUtils.<I>newValueReader(schema,
        EDGE_SOURCE_ID_COLUMN, conf, GraphType.VERTEX_ID,
        GiraphHiveConstants.VERTEX_ID_READER_JYTHON_NAME);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
  @Override
  public void initializeRecords(Iterator<HiveReadableRecord> records) {
    super.initializeRecords(records);

    HiveTableSchema schema = getTableSchema();
    ImmutableClassesGiraphConfiguration<I, V, E> conf = getConf();

    vertexIdReader = HiveJythonUtils.newValueReader(schema, VERTEX_ID_COLUMN,
        conf, GraphType.VERTEX_ID,
        GiraphHiveConstants.VERTEX_ID_READER_JYTHON_NAME);
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
      table = client.get_table(inputDesc.getDbName(), inputDesc.getTableName());
    } catch (Exception e) {
      throw new IOException(e);
    }

    final HiveTableSchema tableSchema = HiveTableSchemaImpl.fromTable(table);
    HiveTableSchemas.put(conf, myProfileId, tableSchema);

    List<InputPartition> partitions = computePartitions(inputDesc, client, table);

    List<InputSplit> splits = computeSplits(conf, inputDesc, tableSchema, partitions);
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
      outputInfo.setFinalOutputPath(outputInfo.getPartitionPath());
    } else {
      outputInfo.setFinalOutputPath(table.getSd().getLocation());
    }

    HiveTableSchema tableSchema = HiveTableSchemaImpl.fromTable(table);
    HiveTableSchemas.put(conf, profileId, tableSchema);

    OutputConf outputConf = new OutputConf(conf, profileId);
    outputConf.writeOutputDescription(outputDesc);
    outputConf.writeOutputTableInfo(outputInfo);
View Full Code Here
TOP

Related Classes of com.facebook.hiveio.schema.HiveTableSchema

  • com.facebook.hiveio.input.HiveApiInputFormat
  • com.facebook.hiveio.input.HiveInput
  • com.facebook.hiveio.mapreduce.output.HiveTools
  • com.facebook.hiveio.output.HiveApiOutputFormat
  • com.facebook.hiveio.output.OutputTest
  • com.facebook.hiveio.output.TypeUpgradeTest
  • com.facebook.hiveio.tailer.Tailer
  • com.facebook.hiveio.tailer.TailerCmd
  • org.apache.giraph.hive.input.CheckInputTest
  • org.apache.giraph.hive.input.edge.HiveEdgeInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.