Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataInputStream.readUTF()

118
119
120
121
122
123
124
125
126
127
128
      //opening clueweb forward index files
      docForwardIndex = new DocumentForwardIndex[clueweb.size()];

      for(int i = 0; i < clueweb.size(); i++) {
        in = FileSystem.get(conf).open(new Path(clueweb.get(i)));
        String indexClueWebClass = in.readUTF();
        in.close();
        try {
          docForwardIndex[i] = (DocumentForwardIndex<Indexable>) Class.forName(indexClueWebClass).newInstance();
          docForwardIndex[i].loadIndex(new Path(clueweb.get(i)), new Path(mappingFile), FileSystem.get(conf));
          lastDocs[i] = docForwardIndex[i].getLastDocno();
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    docnoMapping.loadMapping(mapping, fs);

    FSDataInputStream in = fs.open(index);

    // class name; throw away
    in.readUTF();
    collectionPath = in.readUTF();

    int blocks = in.readInt();

    docnos = new int[blocks];
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
    FSDataInputStream in = fs.open(index);

    // class name; throw away
    in.readUTF();
    collectionPath = in.readUTF();

    int blocks = in.readInt();

    docnos = new int[blocks];
    offsets = new int[blocks];
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    int sz = in.readInt() + 1;
    LOG.warn("creating array of length: " + sz);
    String[] arr = new String[sz];

    for (int i = 1; i < sz; i++) {
      arr[i] = in.readUTF();
    }
    in.close();

    // Can't leave the zero'th entry null, or else we might get a null pointer exception during a
    // binary search on the array.
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
    for (int i = 0; i < 105 * 50; i++) {
      offsets[i] = -1;
    }

    for (int i = 0; i < sz; i++) {
      String docName = in.readUTF();

      // WTX082-B50-226
      int dirNum = Integer.parseInt(docName.substring(3, 6));
      int subdirNum = Integer.parseInt(docName.substring(8, 10));
      int num = Integer.parseInt(docName.substring(11));
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    for (int i = 0; i < 273 * 100; i++) {
      offsets[i] = -1;
    }

    for (int i = 0; i < sz; i++) {
      String docName = in.readUTF();

      // GX243-38-13543987
      int dirNum = Integer.parseInt(docName.substring(2, 5));
      int subdirNum = Integer.parseInt(docName.substring(6, 8));
      int num = Integer.parseInt(docName.substring(9));
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
  @Override
  public void loadIndex(Path index, Path mapping, FileSystem fs) throws IOException {
    FSDataInputStream in = fs.open(index);

    // Read and throw away.
    in.readUTF();
    collectionPath = in.readUTF();

    // Docnos start at one, so we need an array that's one larger than number of docs.
    int sz = in.readInt() + 1;
    offsets = new long[sz];
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
  public void loadIndex(Path index, Path mapping, FileSystem fs) throws IOException {
    FSDataInputStream in = fs.open(index);

    // Read and throw away.
    in.readUTF();
    collectionPath = in.readUTF();

    // Docnos start at one, so we need an array that's one larger than number of docs.
    int sz = in.readInt() + 1;
    offsets = new long[sz];
    lengths = new int[sz];
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
  @Override
  public void loadIndex(Path index, Path mapping, FileSystem fs) throws IOException {
    FSDataInputStream in = fs.open(index);

    // Read and throw away.
    in.readUTF();
    path = in.readUTF();

    // Docnos start at one, so we need an array that's one larger than number of docs.
    int sz = in.readInt() + 1;
    offsets = new long[sz];
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
  public void loadIndex(Path index, Path mapping, FileSystem fs) throws IOException {
    FSDataInputStream in = fs.open(index);

    // Read and throw away.
    in.readUTF();
    path = in.readUTF();

    // Docnos start at one, so we need an array that's one larger than number of docs.
    int sz = in.readInt() + 1;
    offsets = new long[sz];
    lengths = new int[sz];
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.