Examples of org.commoncrawl.protocol.shared.ArcFileHeaderItem


      while ((line = reader.readLine()) != null) {
        if (line.length() != 0) {
          int colonPos = line.indexOf(':');


          ArcFileHeaderItem item = new ArcFileHeaderItem();


          if (colonPos != -1 && colonPos != line.length() - 1) {


            item.setItemKey(line.substring(0, colonPos));
            item.setItemValue(line.substring(colonPos + 1));


            // if this is our special truncation flag ...
            if (item.getItemKey().equals(
                Constants.ARCFileHeader_ContentTruncated)) {
              String parts[] = item.getItemValue().split(",");
              for (String part : parts) {
                if (part.equals(ArcFileItem.Flags
                    .toString(ArcFileItem.Flags.TruncatedInInflate))) {
                  _item.setFlags(_item.getFlags()
                      | ArcFileItem.Flags.TruncatedInDownload);
                } else if (part.equals(ArcFileItem.Flags
                    .toString(ArcFileItem.Flags.TruncatedInInflate))) {
                  _item.setFlags(_item.getFlags()
                      | ArcFileItem.Flags.TruncatedInInflate);
                }
              }
            }
          } else {
            item.setItemValue(line);
          }
          _item.getHeaderItems().add(item);
        }
      }
    }

    //populate headers 
    for (int i=0;i<headers.getKeyCount();++i) { 
      String headerKey = headers.getKey(i);
      String headerValue = headers.getValue(i);
      
      ArcFileHeaderItem headerItem = new ArcFileHeaderItem();
      
      headerItem.setItemKey((headerKey != null) ? headerKey : "");
      headerItem.setItemValue((headerValue != null) ? headerValue : "");
      
      arcFileItem.getHeaderItems().add(headerItem);
    }
    // set up buffer pointer ... 
    arcFileItem.setContent(new FlexBuffer(rawArcPayload.getBytes(),headerLen,contentLen,true));

Examples of org.commoncrawl.protocol.shared.ArcFileHeaderItem

Related Classes of org.commoncrawl.protocol.shared.ArcFileHeaderItem