Package water.fvec

Examples of water.fvec.Frame$Slice


   *         one column with predicted values.
   */
  public final Frame score(Frame fr, boolean adapt) {
    int ridx = fr.find(responseName());
    if (ridx != -1) { // drop the response for scoring!
      fr = new Frame(fr);
      fr.remove(ridx);
    }
    // Adapt the Frame layout - returns adapted frame and frame containing only
    // newly created vectors
    Frame[] adaptFrms = adapt ? adapt(fr,false) : null;
    // Adapted frame containing all columns - mix of original vectors from fr
    // and newly created vectors serving as adaptors
    Frame adaptFrm = adapt ? adaptFrms[0] : fr;
    // Contains only newly created vectors. The frame eases deletion of these vectors.
    Frame onlyAdaptFrm = adapt ? adaptFrms[1] : null;
    // Invoke scoring
    Frame output = scoreImpl(adaptFrm);
    // Be nice to DKV and delete vectors which i created :-)
    if (adapt) onlyAdaptFrm.delete();
    return output;
  }
View Full Code Here


   *  It returns a <b>two element array</b> containing an adapted frame and a
   *  frame which contains only vectors which where adapted (the purpose of the
   *  second frame is to delete all adapted vectors with deletion of the
   *  frame). */
  public Frame[] adapt( final Frame fr, boolean exact) {
    Frame vfr = new Frame(fr); // To avoid modification of original frame fr
    int ridx = vfr.find(_names[_names.length-1]);
    if(ridx != -1 && ridx != vfr._names.length-1){ // Unify frame - put response to the end
      String n = vfr._names[ridx];
      vfr.add(n,vfr.remove(ridx));
    }
    int n = ridx == -1?_names.length-1:_names.length;
    String [] names = Arrays.copyOf(_names, n);
    Frame  [] subVfr;
    // replace missing columns with NaNs (or 0s for DeepLearning with sparse data)
    // subVfr = vfr.subframe(names, (this instanceof DeepLearningModel && ((DeepLearningModel)this).get_params().sparse) ? 0 : Double.NaN);
    subVfr = vfr.subframe(names, Double.NaN);
    vfr = subVfr[0]; // extract only subframe but keep the rest for delete later
    Vec[] frvecs = vfr.vecs();
    boolean[] toEnum = new boolean[frvecs.length];
    if(!exact) for(int i = 0; i < n;++i)
      if(_domains[i] != null && !frvecs[i].isEnum()) {// if model expects domain but input frame does not have domain => switch vector to enum
        frvecs[i] = frvecs[i].toEnum();
        toEnum[i] = true;
      }
    int[][][] map = adapt(names,vfr.domains(),exact);
    assert map.length == names.length; // Be sure that adapt call above do not skip any column
    ArrayList<Vec> avecs = new ArrayList<>(); // adapted vectors
    ArrayList<String> anames = new ArrayList<>(); // names for adapted vector

    for( int c=0; c<map.length; c++ ) // Iterate over columns
      if(map[c] != null) { // Column needs adaptation
        Vec adaptedVec;
        if (toEnum[c]) { // Vector was flipped to column already, compose transformation
          adaptedVec = TransfVec.compose((TransfVec) frvecs[c], map[c], vfr.domains()[c], false);
        } else adaptedVec = frvecs[c].makeTransf(map[c], vfr.domains()[c]);
        avecs.add(frvecs[c] = adaptedVec);
        anames.add(names[c]); // Collect right names
      } else if (toEnum[c]) { // Vector was transformed to enum domain, but does not need adaptation we need to record it
        avecs.add(frvecs[c]);
        anames.add(names[c]);
      }
    // Fill trash bin by vectors which need to be deleted later by the caller.
    Frame vecTrash = new Frame(anames.toArray(new String[anames.size()]), avecs.toArray(new Vec[avecs.size()]));
//    if (subVfr[1]!=null) vecTrash.add(subVfr[1], true);
    return new Frame[] { new Frame(names,frvecs), vecTrash };
  }
View Full Code Here

    public KeyInfo(Key k, Value v){
      _key = k;
      _type = v.type();
      if(v.isFrame()) {
        Frame f = v.get();
        _rawData = (f.vecs().length == 1 && (f.anyVec() instanceof ByteVec));
      } else _rawData = false;
      _sz = v._max;
      _backEnd = v.backend();
    }
View Full Code Here

   * Internal method called from <code>drmFromFile</code> if format verified.
   */
  public static H2ODrm drmFromSeqfile(String filename, int parMin) {
    long rows = 0;
    int cols = 0;
    Frame frame = null;
    Vec labels = null;

    SequenceFile.Reader reader = null;
    try {
      String uri = filename;
      Configuration conf = new Configuration();
      Path path = new Path(uri);
      FileSystem fs = FileSystem.get(URI.create(uri), conf);
      Vec.Writer writers[];
      Vec.Writer labelwriter = null;
      boolean isIntKey = false, isLongKey = false, isStringKey = false;

      reader = new SequenceFile.Reader(fs, path, conf);

      if (reader.getValueClass() != VectorWritable.class) {
        System.out.println("ValueClass in file " + filename +
                           "must be VectorWritable, but found " +
                           reader.getValueClassName());
        return null;
      }

      Writable key = (Writable)
        ReflectionUtils.newInstance(reader.getKeyClass(), conf);
      VectorWritable value = (VectorWritable)
        ReflectionUtils.newInstance(reader.getValueClass(), conf);

      long start = reader.getPosition();

      if (reader.getKeyClass() == Text.class) {
        isStringKey = true;
      } else if (reader.getKeyClass() == LongWritable.class) {
        isLongKey = true;
      } else {
        isIntKey = true;
      }

      while (reader.next(key, value)) {
        if (cols == 0) {
          Vector v = value.get();
          cols = Math.max(v.size(), cols);
        }
        if (isLongKey) {
          rows = Math.max(((LongWritable)(key)).get()+1, rows);
        }
        if (isIntKey) {
          rows = Math.max(((IntWritable)(key)).get()+1, rows);
        }
        if (isStringKey) {
          rows++;
        }
      }
      reader.seek(start);

      frame = H2OHelper.emptyFrame(rows, cols, parMin, -1);
      writers = new Vec.Writer[cols];
      for (int i = 0; i < writers.length; i++) {
        writers[i] = frame.vecs()[i].open();
      }

      if (reader.getKeyClass() == Text.class) {
        labels = frame.anyVec().makeZero();
        labelwriter = labels.open();
      }

      long r = 0;
      while (reader.next(key, value)) {
View Full Code Here

   *
   * @param filename Filename to create and store DRM data in.
   * @param drm DRM object storing Matrix data in memory.
   */
  public static void drmToFile(String filename, H2ODrm drm) throws java.io.IOException {
    Frame frame = drm.frame;
    Vec labels = drm.keys;
    String uri = filename;
    Configuration conf = new Configuration();
    Path path = new Path(uri);
    FileSystem fs = FileSystem.get(URI.create(uri), conf);
    SequenceFile.Writer writer = null;
    boolean isSparse = H2OHelper.isSparse(frame);
    ValueString vstr = new ValueString();

    if (labels != null) {
      writer = SequenceFile.createWriter(fs, conf, path, Text.class, VectorWritable.class);
    } else {
      writer = SequenceFile.createWriter(fs, conf, path, IntWritable.class, VectorWritable.class);
    }

    for (long r = 0; r < frame.anyVec().length(); r++) {
      Vector v = null;
      if (isSparse) {
        v = new SequentialAccessSparseVector(frame.numCols());
      } else {
        v = new DenseVector(frame.numCols());
      }

      for (int c = 0; c < frame.numCols(); c++) {
        v.setQuick(c, frame.vecs()[c].at(r));
      }

      if (labels != null) {
        writer.append(new Text(labels.atStr(vstr, r).toString()), new VectorWritable(v));
      } else {
View Full Code Here

   * @param drmA DRM representing matrix A.
   * @param B in-core Mahout Matrix.
   * @return new DRM containing drmA times B.
   */
  public static H2ODrm exec(H2ODrm drmA, Matrix B) {
    Frame A = drmA.frame;
    Vec keys = drmA.keys;
    Frame AinCoreB = null;

    if (B instanceof DiagonalMatrix) {
      AinCoreB = execDiagonal(A, B.viewDiagonal());
    } else {
      AinCoreB = execCommon(A, B);
View Full Code Here

   * @param drmA DRM representing matrix A.
   * @param x in-core Mahout Vector.
   * @return new DRM containing Ax.
   */
  public static H2ODrm exec(H2ODrm drmA, Vector x) {
    Frame A = drmA.frame;
    Vec keys = drmA.keys;
    final H2OBCast<Vector> bx = new H2OBCast<Vector>(x);

    // Ax is written into nc (single element, not array) with an MRTask on A,
    // and therefore will be similarly partitioned as A.
    //
    // x.size() == A.numCols() == chks.length
    Frame Ax = new MRTask() {
        public void map(Chunk chks[], NewChunk nc) {
          int chunkSize = chks[0].len();
          Vector x = bx.value();

          for (int r = 0; r < chunkSize; r++) {
View Full Code Here

   * @param drmA Input DRM.
   * @param R Range object specifying the start and end row numbers to filter.
   * @return new DRM with just the filtered rows.
   */
  public static H2ODrm exec(H2ODrm drmA, final Range R) {
    Frame A = drmA.frame;
    Vec keys = drmA.keys;

    // Run a filtering MRTask on A. If row number falls within R.start() and
    // R.end(), then the row makes it into the output
    Frame Arr = new MRTask() {
        public void map(Chunk chks[], NewChunk ncs[]) {
          int chunkSize = chks[0].len();
          long chunkStart = chks[0].start();

          // First check if the entire chunk even overlaps with R
View Full Code Here

   * @param drmB DRM representing matrix B.
   * @param op Element-wise operator encoded as a String.
   * @return new DRM containing A (element-wise) B.
   */
  public static H2ODrm exec(H2ODrm drmA, H2ODrm drmB, final String op) {
    final Frame A = drmA.frame;
    final Frame B = drmB.frame;
    Vec keys = drmA.keys;
    int AewB_cols = A.numCols();

    // AewB is written into ncs[] with an MRTask on A, and therefore will
    // be similarly partitioned as A.
    //
    // B may or may not be similarly partitioned as A, but must have the
    // same dimensions of A.
    Frame AewB = new MRTask() {
        private double opfn(String op, double a, double b) {
          if (a == 0.0 && b == 0.0) {
            return 0.0;
          }
          if (op.equals("+")) {
View Full Code Here

   * @param drmA DRM representing matrix A.
   * @param drmB DRM representing matrix B.
   * @return new DRM containing rows of B below A.
   */
  public static H2ODrm exec(H2ODrm drmA, H2ODrm drmB) {
    final Frame fra = drmA.frame;
    final Vec keysa = drmA.keys;
    final Frame frb = drmB.frame;
    final Vec keysb = drmB.keys;

    // Create new frame and copy A's data at the top, and B's data below.
    // Create the frame in the same VectorGroup as A, so A's data does not
    // cross the wire during copy. B's data could potentially cross the wire.
    Frame frbind = H2OHelper.emptyFrame(fra.numRows() + frb.numRows(), fra.numCols(),
            -1, -1, fra.anyVec().group());
    Vec keys = null;

    MRTask task = new MRTask() {
        public void map(Chunk chks[], NewChunk nc) {
View Full Code Here

TOP

Related Classes of water.fvec.Frame$Slice

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.