Package org.apache.hive.hcatalog.data.transfer

Examples of org.apache.hive.hcatalog.data.transfer.ReaderContext

62
63
64
65
66
67
68
69
70
71
72
73
74
75
  public ReaderContext prepareRead() throws HCatException {
    try {
      Job job = new Job(conf);
      HCatInputFormat hcif = HCatInputFormat.setInput(
        job, re.getDbName(), re.getTableName()).setFilter(re.getFilterString());
      ReaderContext cntxt = new ReaderContext();
      cntxt.setInputSplits(hcif.getSplits(
          ShimLoader.getHadoopShims().getHCatShim().createJobContext(job.getConfiguration(), null)));
      cntxt.setConf(job.getConfiguration());
      return cntxt;
    } catch (IOException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
    } catch (InterruptedException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    ois.close();

    runsInSlave(cntxt);
    commit(map, true, cntxt);

    ReaderContext readCntxt = runsInMaster(map, false);

    File readCntxtFile = File.createTempFile("hcat-read", "temp");
    readCntxtFile.deleteOnExit();
    oos = new ObjectOutputStream(new FileOutputStream(readCntxtFile));
    oos.writeObject(readCntxt);
    oos.flush();
    oos.close();

    ois = new ObjectInputStream(new FileInputStream(readCntxtFile));
    readCntxt = (ReaderContext) ois.readObject();
    ois.close();

    for (InputSplit split : readCntxt.getSplits()) {
      runsInSlave(split, readCntxt.getConf());
    }
  }
View Full Code Here
111
112
113
114
115
116
117
118
  private ReaderContext runsInMaster(Map<String, String> config, boolean bogus)
    throws HCatException {
    ReadEntity entity = new ReadEntity.Builder().withTable("mytbl").build();
    HCatReader reader = DataTransferFactory.getHCatReader(entity, config);
    ReaderContext cntxt = reader.prepareRead();
    return cntxt;
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    ois.close();

    runsInSlave(cntxt);
    commit(map, true, cntxt);

    ReaderContext readCntxt = runsInMaster(map, false);

    File readCntxtFile = File.createTempFile("hcat-read", "temp");
    readCntxtFile.deleteOnExit();
    oos = new ObjectOutputStream(new FileOutputStream(readCntxtFile));
    oos.writeObject(readCntxt);
    oos.flush();
    oos.close();

    ois = new ObjectInputStream(new FileInputStream(readCntxtFile));
    readCntxt = (ReaderContext) ois.readObject();
    ois.close();

    for (int i = 0; i < readCntxt.numSplits(); i++) {
      runsInSlave(readCntxt, i);
    }
  }
View Full Code Here
111
112
113
114
115
116
117
118
  private ReaderContext runsInMaster(Map<String, String> config, boolean bogus)
    throws HCatException {
    ReadEntity entity = new ReadEntity.Builder().withTable("mytbl").build();
    HCatReader reader = DataTransferFactory.getHCatReader(entity, config);
    ReaderContext cntxt = reader.prepareRead();
    return cntxt;
  }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
public class DataReaderSlave {

  public static void main(String[] args) throws IOException, ClassNotFoundException {

    ObjectInputStream ois = new ObjectInputStream(new FileInputStream(new File(args[0])));
    ReaderContext cntxt = (ReaderContext) ois.readObject();
    ois.close();

    String[] inpSlitsToRead = args[1].split(",");
    List<InputSplit> splits = cntxt.getSplits();

    for (int i = 0; i < inpSlitsToRead.length; i++) {
      InputSplit split = splits.get(Integer.parseInt(inpSlitsToRead[i]));
      HCatReader reader = DataTransferFactory.getHCatReader(split, cntxt.getConf());
      Iterator<HCatRecord> itr = reader.read();
      File f = new File(args[2] + "-" + i);
      f.delete();
      BufferedWriter outFile = new BufferedWriter(new FileWriter(f));
      while (itr.hasNext()) {
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
    for (Entry<Object, Object> kv : externalConfigs.entrySet()) {
      config.put((String) kv.getKey(), (String) kv.getValue());
    }

    // This piece of code runs in master node and gets necessary context.
    ReaderContext context = runsInMaster(config);

    ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(new File(args[1])));
    oos.writeObject(context);
    oos.flush();
    oos.close();
View Full Code Here
63
64
65
66
67
68
69
70
  private static ReaderContext runsInMaster(Map<String, String> config) throws HCatException {

    ReadEntity.Builder builder = new ReadEntity.Builder();
    ReadEntity entity = builder.withTable(config.get("table")).build();
    HCatReader reader = DataTransferFactory.getHCatReader(entity, config);
    ReaderContext cntxt = reader.prepareRead();
    return cntxt;
  }
View Full Code Here
TOP

Related Classes of org.apache.hive.hcatalog.data.transfer.ReaderContext

  • org.apache.hadoop.conf.Configuration
  • org.apache.hive.hcatalog.data.TestReaderWriter
  • org.apache.hive.hcatalog.data.transfer.impl.HCatInputFormatReader
  • org.apache.hive.hcatalog.mapreduce.HCatSplit
  • org.apache.hive.hcatalog.utils.DataReaderMaster
  • org.apache.hive.hcatalog.utils.DataReaderSlave
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.