Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataOutputStream$Buffer


    {
      fs.mkdirs(new Path(store));
    }
   
    HashMap<String,String> params=new HashMap<String, String>();
    FSDataOutputStream out=fs.create(new Path(store,String.valueOf(System.currentTimeMillis())));
      OutputStreamWriter osw = new OutputStreamWriter(out, "UTF-8");

    Upload up=new Upload();
    up.mergerTo(request, response, "gbk", osw, params);
    osw.close();

    out.close();
   
    outStream.append(store);
    outStream.append("<br>\r\n");
    for(Entry<String, String> e:params.entrySet())
    {
View Full Code Here


      fs.delete(new Path(solrpath,"conf/schema.xml"),true);
    }
   
   
   
    FSDataOutputStream out=fs.create(new Path(solrpath,"conf/schema.xml"));
    java.io.BufferedWriter writer=new BufferedWriter(new OutputStreamWriter(out));
    writer.append(topschema);
    writer.append(buffer.toString());
    writer.append(downschema);
    writer.close();
    out.close();
   
    System.out.println(buffer.toString());
    System.out.println("create succ at "+tablepath.toString());

    System.exit(0);
View Full Code Here

    if(ishdfsmode)
    {
      if(!lfs.exists(indexlinks))
      {
        IndexUtils.truncate(lfs, localPartionWork);
        FSDataOutputStream outlinks = lfs.create(indexlinks);
        outlinks.write((new String("@hdfs@"+hdfsPartionShardPath.toString()  + "\r\n")).getBytes());
        outlinks.close();
      }
    }else
    {
      boolean isnotrealtime=false;
      try{
      if(fs.exists(hdfsPartionShardPath))
      {
        FileStatus fstat=fs.getFileStatus(hdfsPartionShardPath);//针对云梯,要判断是文件还是目录
        if(!fstat.isDir())
        {
          isnotrealtime=true;
        }
      }else{
        isnotrealtime=true;
      }
      }catch(Throwable e){
        isnotrealtime=true;
      }
      if(isrealtime||(!isnotrealtime&&(fs.exists(realtime)||lfs.exists(localrealtime))))
      {
        //TODO nothing
      }else//离线模式
      {
        boolean iscopy = false;
        String partionDisk = IndexUtils.getPath(this.diskDirList, taskIndex,key.hashCode(), this.lfs);
        Path localPartionStorePath = new Path(new Path(partionDisk, "higo"),tablename + "/" + this.params.compname + "_" + this.taskIndex + "/" + key);
        iscopy=IndexUtils.copyToLocal(fs, lfs, hdfsPartionShardPath,localPartionStorePath,new Path(partionDisk, "higotmp/"+tablename + "/" + this.params.compname + "_" + this.taskIndex ),true);
        if(!iscopy)
        {
          copy2indexFailTimes.incrementAndGet();
        }
     
        if(iscopy||!lfs.exists(indexlinks))
        {
          IndexUtils.truncate(lfs, localPartionWork);
          FSDataOutputStream outlinks = lfs.create(indexlinks);
          outlinks.write((new String(localPartionStorePath.toString()  + "\r\n")).getBytes());
          outlinks.close();
        }
      }
    }
   
    Path hdfsvertify=new Path(hdfspartion, "vertify");
View Full Code Here

        // disable recovery if this is a restart
        shouldRecover = false;

        // write the jobtracker.info file
        try {
          FSDataOutputStream out = FileSystem.create(fs, restartFile,
                                                     filePerm);
          out.writeInt(0);
          out.close();
        } catch (IOException ioe) {
          LOG.warn("Writing to file " + restartFile + " failed!");
          LOG.warn("FileSystem is not ready yet!");
          fs.delete(restartFile, false);
          throw ioe;
        }
        return;
      }

      FSDataInputStream in = fs.open(restartFile);
      try {
        // read the old count
        restartCount = in.readInt();
        ++restartCount; // increment the restart count
      } catch (IOException ioe) {
        LOG.warn("System directory is garbled. Failed to read file "
                 + restartFile);
        LOG.warn("Jobtracker recovery is not possible with garbled"
                 + " system directory! Please delete the system directory and"
                 + " restart the jobtracker. Note that deleting the system"
                 + " directory will result in loss of all the running jobs.");
        throw new RuntimeException(ioe);
      } finally {
        if (in != null) {
          in.close();
        }
      }

      // Write back the new restart count and rename the old info file
      //TODO This is similar to jobhistory recovery, maybe this common code
      //      can be factored out.
     
      // write to the tmp file
      FSDataOutputStream out = FileSystem.create(fs, tmpRestartFile, filePerm);
      out.writeInt(restartCount);
      out.close();

      // delete the main file
      fs.delete(restartFile, false);
     
      // rename the .rec to main file
View Full Code Here

      if (!recovered) {
        // Store the information in a file so that the job can be recovered
        // later (if at all)
        Path jobDir = getSystemDirectoryForJob(jobId);
        FileSystem.mkdirs(fs, jobDir, new FsPermission(SYSTEM_DIR_PERMISSION));
        FSDataOutputStream out = fs.create(getSystemFileForJob(jobId));
        jobInfo.write(out);
        out.close();
      }
      return addJob(jobId, job);
    }
  }
View Full Code Here

      LOG.info("Recovering file");
      recoverFile(fs2);

      LOG.info("Opening file for append from new fs");
      FSDataOutputStream appenderStream = fs2.append(file1);
     
      LOG.info("Writing some data from new appender");
      AppendTestUtil.write(appenderStream, 0, 4096);
     
      LOG.info("Telling old close to proceed.");
      delayer.proceed();
      LOG.info("Waiting for close to finish.");
      t.join();
      LOG.info("Close finished.");

      // We expect that close will get a "Lease mismatch"
      // error.
      Throwable thrownByClose = err.get();
      assertNotNull(thrownByClose);
      assertTrue(thrownByClose instanceof IOException);
      if (!thrownByClose.getMessage().contains(
            "Lease mismatch"))
        throw thrownByClose;
     
      // The appender should be able to close properly
      appenderStream.close();
    } finally {
      cluster.shutdown();
    }
  } 
View Full Code Here

        Path dir = new Path("/dir" + i);
        hdfs.mkdirs(dir);
        writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
        for(int j = 0; j < FILES_PER_DIR; j++) {
          Path file = new Path(dir, "file" + j);
          FSDataOutputStream o = hdfs.create(file);
          o.write(new byte[ filesize++ ]);
          o.close();
         
          writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString()));
        }
      }
View Full Code Here

     
      FileSystem fs = path.getFileSystem(getConf());
      LOG.info("Writing to fs: " + fs);
     
      while (true) {
        FSDataOutputStream stm = fs.create(path, true);
 
        Syncer syncer = new Syncer(stm);
        syncer.start();
 
        byte b[] = THING_TO_WRITE;
        long start = System.currentTimeMillis();
        long lastPrint = System.currentTimeMillis();
        while (syncer.isAlive()) {
          stm.write(b);
          curIteration++;
          if (System.currentTimeMillis() - lastPrint > 1000) {
            LOG.info("Written " + curIteration + " writes");
            lastPrint = System.currentTimeMillis();
          }
        }
        if (err.get() != null)
          throw new RuntimeException("Failed", err.get());
 
        syncer.stopSyncing();
        syncer.join();
        stm.close();
 
        if (err.get() != null)
          throw new RuntimeException("Failed", err.get());
      }
    }
View Full Code Here

   * except as arguments to the q option.
   */
  static Varargs format( Varargs args ) {
    LuaString fmt = args.checkstring( 1 );
    final int n = fmt.length();
    Buffer result = new Buffer(n);
    int arg = 1;
    int c;
   
    for ( int i = 0; i < n; ) {
      switch ( c = fmt.luaByte( i++ ) ) {
      case '\n':
        result.append( "\n" );
        break;
      default:
        result.append( (byte) c );
        break;
      case L_ESC:
        if ( i < n ) {
          if ( ( c = fmt.luaByte( i ) ) == L_ESC ) {
            ++i;
            result.append( (byte)L_ESC );
          } else {
            arg++;
            FormatDesc fdsc = new FormatDesc(args, fmt, i );
            i += fdsc.length;
            switch ( fdsc.conversion ) {
            case 'c':
              fdsc.format( result, (byte)args.checkint( arg ) );
              break;
            case 'i':
            case 'd':
              fdsc.format( result, args.checkint( arg ) );
              break;
            case 'o':
            case 'u':
            case 'x':
            case 'X':
              fdsc.format( result, args.checklong( arg ) );
              break;
            case 'e':
            case 'E':
            case 'f':
            case 'g':
            case 'G':
              fdsc.format( result, args.checkdouble( arg ) );
              break;
            case 'q':
              addquoted( result, args.checkstring( arg ) );
              break;
            case 's': {
              LuaString s = args.checkstring( arg );
              if ( fdsc.precision == -1 && s.length() >= 100 ) {
                result.append( s );
              } else {
                fdsc.format( result, s );
              }
            }  break;
            default:
              error("invalid option '%"+(char)fdsc.conversion+"' to 'format'");
              break;
            }
          }
        }
      }
    }
   
    return result.tostring();
  }
View Full Code Here

    LuaString p = args.checkstring( 2 );
    LuaValue repl = args.arg( 3 );
    int max_s = args.optint( 4, srclen + 1 );
    final boolean anchor = p.length() > 0 && p.charAt( 0 ) == '^';
   
    Buffer lbuf = new Buffer( srclen );
    MatchState ms = new MatchState( args, src, p );
   
    int soffset = 0;
    int n = 0;
    while ( n < max_s ) {
      ms.reset();
      int res = ms.match( soffset, anchor ? 1 : 0 );
      if ( res != -1 ) {
        n++;
        ms.add_value( lbuf, soffset, res, repl );
      }
      if ( res != -1 && res > soffset )
        soffset = res;
      else if ( soffset < srclen )
        lbuf.append( (byte) src.luaByte( soffset++ ) );
      else
        break;
      if ( anchor )
        break;
    }
    lbuf.append( src.substring( soffset, srclen ) );
    return varargsOf(lbuf.tostring(), valueOf(n));
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.FSDataOutputStream$Buffer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.