Package org.apache.hcatalog.data.transfer

Examples of org.apache.hcatalog.data.transfer.ReaderContext

48
49
50
51
52
53
54
55
56
57
58
    for (Entry<Object, Object> kv : externalConfigs.entrySet()){
      config.put((String)kv.getKey(), (String)kv.getValue());
    }
   
    // This piece of code runs in master node and gets necessary context.
    ReaderContext context = runsInMaster(config);

    ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(new File(args[1])));
    oos.writeObject(context);
    oos.flush();
    oos.close();
View Full Code Here
62
63
64
65
66
67
68
69
  private static ReaderContext runsInMaster(Map<String,String> config) throws HCatException {

    ReadEntity.Builder builder = new ReadEntity.Builder();
    ReadEntity entity = builder.withTable(config.get("table")).build();
    HCatReader reader = DataTransferFactory.getHCatReader(entity, config);
    ReaderContext cntxt = reader.prepareRead();
    return cntxt;
  }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    ois.close();

    runsInSlave(cntxt);
    commit(map, true, cntxt);

    ReaderContext readCntxt = runsInMaster(map, false);

    File readCntxtFile = File.createTempFile("hcat-read", "temp");
    readCntxtFile.deleteOnExit();
    oos = new ObjectOutputStream(new FileOutputStream(readCntxtFile));
    oos.writeObject(readCntxt);
    oos.flush();
    oos.close();

    ois = new ObjectInputStream(new FileInputStream(readCntxtFile));
    readCntxt = (ReaderContext) ois.readObject();
    ois.close();

    for (InputSplit split : readCntxt.getSplits()) {
      runsInSlave(split, readCntxt.getConf());
    }
  }
View Full Code Here
114
115
116
117
118
119
120
121
  private ReaderContext runsInMaster(Map<String, String> config, boolean bogus)
    throws HCatException {
    ReadEntity entity = new ReadEntity.Builder().withTable("mytbl").build();
    HCatReader reader = DataTransferFactory.getHCatReader(entity, config);
    ReaderContext cntxt = reader.prepareRead();
    return cntxt;
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  public ReaderContext prepareRead() throws HCatException {
    try {
      Job job = new Job(conf);
      HCatInputFormat hcif = HCatInputFormat.setInput(
        job, re.getDbName(), re.getTableName()).setFilter(re.getFilterString());
      ReaderContext cntxt = new ReaderContext();
      cntxt.setInputSplits(hcif.getSplits(
        ShimLoader.getHadoopShims().getHCatShim().createJobContext(job.getConfiguration(), null)));
      cntxt.setConf(job.getConfiguration());
      return cntxt;
    } catch (IOException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
    } catch (InterruptedException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
View Full Code Here
TOP

Related Classes of org.apache.hcatalog.data.transfer.ReaderContext

  • org.apache.hadoop.conf.Configuration
  • org.apache.hcatalog.data.TestReaderWriter
  • org.apache.hcatalog.data.transfer.impl.HCatInputFormatReader
  • org.apache.hcatalog.mapreduce.HCatSplit
  • org.apache.hcatalog.utils.DataReaderMaster
  • org.apache.hcatalog.utils.DataReaderSlave
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.