Examples of org.apache.avro.util.Utf8$Utf8Converter

org.apache.avro.util.Utf8
A Utf8 string. Unlike {@link String}, instances are mutable. This is more efficient than {@link String} when reading or writing a sequence of values,as a single instance may be reused.

      // now add the metadata
      Iterator<String> keysIter = metadata.keySet().iterator();
      while (keysIter.hasNext()) {
          String keymd = keysIter.next();
          String valuemd = metadata.get(keymd);
          row.putToMetadata(new Utf8(keymd), ByteBuffer.wrap(valuemd.getBytes()));
      }


      if (customScore != -1)
        row.setScore(customScore);
      else

View Full Code Here

  public NutchDocument filter(NutchDocument doc, String url, WebPage page)
      throws IndexingException {


    // check if LANGUAGE found, possibly put there by HTMLLanguageParser
    String lang = null;
    ByteBuffer blang = page.getFromMetadata(new Utf8(Metadata.LANGUAGE));
    if (blang != null) {
      lang = Bytes.toString(blang.array());
    }


    if (lang == null || lang.length() == 0) {

View Full Code Here


    if (page.getInlinks() != null) {
      page.getInlinks().clear();
    }
    for (ScoreDatum inlink : inlinkedScoreData) {
      page.putToInlinks(new Utf8(inlink.getUrl()), new Utf8(inlink.getAnchor()));
    }


    try {
      scoringFilters.updateScore(url, page, inlinkedScoreData);
    } catch (ScoringFilterException e) {
      LOG.warn("Scoring filters failed with exception " +
                StringUtils.stringifyException(e));
    }


    // clear markers
    // But only delete when they exist. This is much faster for the underlying
    // store. The markers are on the input anyway.
    if (page.getFromMetadata(FetcherJob.REDIRECT_DISCOVERED) != null) {
      page.removeFromMetadata(FetcherJob.REDIRECT_DISCOVERED);
    }
    Mark.GENERATE_MARK.removeMarkIfExist(page);
    Mark.FETCH_MARK.removeMarkIfExist(page);
    Utf8 mark = Mark.PARSE_MARK.removeMarkIfExist(page);
    if (mark != null) {
      Mark.UPDATEDB_MARK.putMark(page, mark);
    }


    context.write(keyUrl, page);

View Full Code Here

    case ENUM:
      return s.getEnumOrdinal(o.toString());
    case NULL:
      return 0;
    case STRING:
      return (o instanceof Utf8 ? o : new Utf8(o.toString())).hashCode();
    default:
      return o.hashCode();
    }
  }

View Full Code Here

        ? compare(o1, o2, s.getTypes().get(i1), equals)
        : i1 - i2;
    case NULL:
      return 0;
    case STRING:
      Utf8 u1 = o1 instanceof Utf8 ? (Utf8)o1 : new Utf8(o1.toString());
      Utf8 u2 = o2 instanceof Utf8 ? (Utf8)o2 : new Utf8(o2.toString());
      return u1.compareTo(u2);
    default:
      return ((Comparable)o1).compareTo(o2);
    }
  }

View Full Code Here

        // Some CharSequence subclasses are mutable, so we still need to make 
        // a copy
        else if (value instanceof Utf8) {
          // Utf8 copy constructor is more efficient than converting 
          // to string and then back to Utf8
          return (T)new Utf8((Utf8)value);
        }
        return (T)new Utf8(value.toString());
      case UNION:
        return deepCopy(
            schema.getTypes().get(resolveUnion(schema, value)), value);
      default:
        throw new AvroRuntimeException(

View Full Code Here

  }


  /** Called to create a string from a default value.  Subclasses may override
   * to use a different string representation.  By default, this calls {@link
   * Utf8#Utf8(String)}.*/
  protected Object createString(String value) { return new Utf8(value); }

View Full Code Here

    fields.add(new Schema.Field("f", TestRecord.SCHEMA$, "", null));
    schema.setFields(fields);


    // create a generic instance of this record
    TestRecord nested = new TestRecord();
    nested.name = new Utf8("foo");
    nested.kind = Kind.BAR;
    nested.hash = new MD5();
    System.arraycopy(new byte[]{0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5}, 0,
                     nested.hash.bytes(), 0, 16);
    GenericData.Record record = new GenericData.Record(schema);

View Full Code Here

  @Test
  public void testBasicStorage() {
    TracePluginConfiguration conf = new TracePluginConfiguration();
    FileSpanStorage test = new FileSpanStorage(false, conf);
    Span s = Util.createEventlessSpan(Util.idValue(1), Util.idValue(1), null);
    s.messageName = new Utf8("message");
    test.addSpan(s);
    try {
      Thread.sleep(100);
    } catch (InterruptedException e) {
      e.printStackTrace();

View Full Code Here

    FileSpanStorage test = new FileSpanStorage(false, conf);
    test.setMaxSpans(100000);
    List<Span> spans = new ArrayList<Span>(50000);
    for (int i = 0; i < 50000; i++) {
      Span s = Util.createEventlessSpan(Util.idValue(i), Util.idValue(i), null);
      s.messageName = new Utf8("message");
      test.addSpan(s);
      spans.add(s);
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
    assertEquals(50000, test.getAllSpans().size());


    // Test fewer spans but explicitly call containsAll
    TracePluginConfiguration conf2 = new TracePluginConfiguration();
    FileSpanStorage test2 = new FileSpanStorage(false, conf2);
    test.setMaxSpans(100000);
    spans.clear();
    for (int i = 0; i < 5000; i++) {
      Span s = Util.createEventlessSpan(Util.idValue(i), Util.idValue(i), null);
      s.messageName = new Utf8("message");
      test2.addSpan(s);
      spans.add(s);
    }
    try {
      Thread.sleep(100);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.avro.util.Utf8$Utf8Converter

avrobase.hs.HSABTest

avrobase.shard.ShardedAvroBaseTest

AvroTest

co.nubetech.hiho.mapreduce.TestDBInputAvroMapper

com.cloudera.cdk.data.hbase.avro.AvroEntityMapperTest

com.cloudera.cdk.data.hbase.avro.io.ColumnDecoderTest

com.cloudera.cdk.data.hbase.avro.ManagedDaoTest

com.cloudera.flume.reporter.server.TestAvroReportServer

com.cloudera.recordbreaker.schemadict.SchemaStatisticalSummary$MapSummaryNode

com.cloudera.sqoop.TestAvroImport

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.