Examples of org.apache.hadoop.io.UTF8

org.apache.hadoop.io.UTF8
A WritableComparable for strings that uses the UTF8 encoding.
Also includes utilities for efficiently reading and writing UTF-8. @deprecated replaced by Text

      FSEditLog.toLogLong(timestamp)};
    logEdit(OP_DELETE, new ArrayWritable(UTF8.class, info), null);
  }
  
  static UTF8 toLogReplication(short replication) {
    return new UTF8(Short.toString(replication));
  }

View Full Code Here

  static short fromLogReplication(UTF8 replication) {
    return Short.parseShort(replication.toString());
  }


  static UTF8 toLogLong(long timestamp) {
    return new UTF8(Long.toString(timestamp));
  }

View Full Code Here


    long totalSize = 0;
    long maxSize = ((megaBytes / numFiles) * 2) + 1;
    try {
      while (totalSize < megaBytes) {
        UTF8 name = new UTF8(Long.toString(random.nextLong()));


        long size = random.nextLong();
        if (size < 0)
          size = -size;
        size = size % maxSize;

View Full Code Here

        out.close();
      }
      // rename to final location
      fs.rename(tempFile, new Path(DATA_DIR, name));


      collector.collect(new UTF8("bytes"), new LongWritable(written));


      reporter.setStatus("wrote " + name);
    }

View Full Code Here

        }
      } finally {
        in.close();
      }


      collector.collect(new UTF8("bytes"), new LongWritable(read));


      reporter.setStatus("read " + name);
    }

View Full Code Here

    // concatenate strings
    if (field.startsWith("s:")) {
      String sSum = "";
      while (values.hasNext())
        sSum += values.next().toString() + ";";
      output.collect(key, new UTF8(sSum));
      reporter.setStatus("finished " + field + " ::host = " + hostName);
      return;
    }
    // sum long values
    if (field.startsWith("f:")) {
      float fSum = 0;
      while (values.hasNext())
        fSum += Float.parseFloat(values.next().toString());
      output.collect(key, new UTF8(String.valueOf(fSum)));
      reporter.setStatus("finished " + field + " ::host = " + hostName);
      return;
    }
    // sum long values
    if (field.startsWith("l:")) {
      long lSum = 0;
      while (values.hasNext()) {
        lSum += Long.parseLong(values.next().toString());
      }
      output.collect(key, new UTF8(String.valueOf(lSum)));
    }
    reporter.setStatus("finished " + field + " ::host = " + hostName);
  }

View Full Code Here

    public void map(WritableComparable key, Text value,
                    OutputCollector<UTF8, UTF8> output,
                    Reporter reporter) throws IOException 
    {
      String line = value.toString();
      output.collect(new UTF8(process(line)), new UTF8(""));    
    }

View Full Code Here

    
    public void reduce(UTF8 key, Iterator<UTF8> values,
                       OutputCollector<UTF8, UTF8> output, Reporter reporter) throws IOException 
    {
      while(values.hasNext()) {
        output.collect(key, new UTF8(values.next().toString()));
      }
    }

View Full Code Here

          throw new AssertionFailedError("Log file does not exist: "+logFile);
        }
    
      // create a directory
      try {
        dfsClient.mkdirs( new UTF8( "/data") );
        assertMkdirs( "/data", false );
      } catch ( IOException ioe ) {
        ioe.printStackTrace();
      }
       
      try {
        dfsClient.mkdirs( new UTF8( "data") );
        assertMkdirs( "data", true );
      } catch ( IOException ioe ) {
         ioe.printStackTrace();
      }
      
      //
      // create a file with 1 data block
      try {
        createFile("/data/xx", 1);
        assertCreate( "/data/xx", 1, false );
      } catch( IOException ioe ) {
      assertCreate( "/data/xx", 1, true );
      }
    
      // create a file with 2 data blocks
      try {
        createFile("/data/yy",BLOCK_SIZE+1);
        assertCreate( "/data/yy", BLOCK_SIZE+1, false );
      } catch( IOException ioe ) {
      assertCreate( "/data/yy", BLOCK_SIZE+1, true );
      }


      // create an existing file
      try {
        createFile("/data/xx", 2);
        assertCreate( "/data/xx", 2, false );
      } catch( IOException ioe ) {
        assertCreate( "/data/xx", 2, true );
      }
    
      // delete the file
      try {
        dfsClient.delete( new UTF8("/data/yy") );
        assertDelete("/data/yy", false);
      } catch( IOException ioe ) {
        ioe.printStackTrace();
      }


    
      // rename the file
      try {
        dfsClient.rename( new UTF8("/data/xx"), new UTF8("/data/yy") );
        assertRename( "/data/xx", "/data/yy", false );
      } catch( IOException ioe ) {
        ioe.printStackTrace();
      }


      try {
        dfsClient.delete(new UTF8("/data/xx"));
        assertDelete("/data/xx", true);
      } catch(IOException ioe) {
      ioe.printStackTrace();
      }
      
      try {
        dfsClient.rename( new UTF8("/data/xx"), new UTF8("/data/yy") );    
        assertRename( "/data/xx", "/data/yy", true );
      } catch( IOException ioe) {
      ioe.printStackTrace();
      }

View Full Code Here

  private void createFile( String filename, long fileSize ) throws IOException { 
    //
    //           write filesize of data to file
    //
    byte[] buffer = new byte[BUFFER_SIZE];
    UTF8 testFileName = new UTF8(filename); // hardcode filename
    FSOutputStream nos;
  nos = dfsClient.create(testFileName, false);
    try {
      for (long nBytesWritten = 0L;
                nBytesWritten < fileSize;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.UTF8

org.apache.hadoop.benchmarks.mapred.BenchmarkMapper

org.apache.hadoop.benchmarks.mapred.BenchmarkReducer

org.apache.hadoop.dfs.ClusterTestDFS

org.apache.hadoop.dfs.ClusterTestDFSNamespaceLogging

org.apache.hadoop.dfs.DatanodeRegistration

org.apache.hadoop.dfs.DataStorage

org.apache.hadoop.dfs.DFSck

org.apache.hadoop.dfs.FSEditLog

org.apache.hadoop.dfs.FSImage

org.apache.hadoop.dfs.NamenodeFsck

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.