Package hipi.imagebundle

Examples of hipi.imagebundle.HipiImageBundle.open()

57
58
59
60
61
62
63
64
65
66
67
    {
      String temp_path = conf.get("downloader.outpath") + key.get() + ".hib.tmp";
      System.out.println("Temp path: " + temp_path);
     
      HipiImageBundle hib = new HipiImageBundle(new Path(temp_path), conf);
      hib.open(HipiImageBundle.FILE_MODE_WRITE, true);

      String word = value.toString();

      BufferedReader reader = new BufferedReader(new StringReader(word));
      String uri;
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
        if(i >= iprev+100) {
          hib.close();
          context.write(new BooleanWritable(true), new Text(hib.getPath().toString()));
          temp_path = conf.get("downloader.outpath") + i + ".hib.tmp";
          hib = new HipiImageBundle(new Path(temp_path), conf);
          hib.open(HipiImageBundle.FILE_MODE_WRITE, true);
          iprev = i;
        }
        long startT=0;
        long stopT=0;    
        startT = System.currentTimeMillis();           
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
    throws IOException, InterruptedException
    {
      if(key.get()){
        FileSystem fileSystem = FileSystem.get(conf);
        HipiImageBundle hib = new HipiImageBundle(new Path(conf.get("downloader.outfile")), conf);
        hib.open(HipiImageBundle.FILE_MODE_WRITE, true);
        for (Text temp_string : values) {
          Path temp_path = new Path(temp_string.toString());
          HipiImageBundle input_bundle = new HipiImageBundle(temp_path, conf);
          hib.append(input_bundle);
         
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
    List<InputSplit> splits = new ArrayList<InputSplit>();
    for (FileStatus file : listStatus(job)) {
      Path path = file.getPath();
      FileSystem fs = path.getFileSystem(conf);
      HipiImageBundle hib = new HipiImageBundle(path, conf);
      hib.open(AbstractImageBundle.FILE_MODE_READ);
      // offset should be guaranteed to be in order
      List<Long> offsets = hib.getOffsets();
      BlockLocation[] blkLocations = fs.getFileBlockLocations(hib.getDataFile(), 0, offsets.get(offsets.size() - 1));
      if (numMapTasks == 0) {
        int i = 0, b = 0;
View Full Code Here
15
16
17
18
19
20
21
22
23
24
25
  public static void main(String[] args) throws IOException {
    File folder = new File(args[0]);
    File[] files = folder.listFiles();
    Configuration conf = new Configuration();
    HipiImageBundle hib = new HipiImageBundle(new Path(args[1]), conf);
    hib.open(AbstractImageBundle.FILE_MODE_WRITE, true);
    for (File file : files) {
      FileInputStream fis = new FileInputStream(file);
      String fileName = file.getName().toLowerCase();
      String suffix = fileName.substring(fileName.lastIndexOf('.'));
      if (suffix.compareTo(".jpg") == 0 || suffix.compareTo(".jpeg") == 0) {
View Full Code Here
18
19
20
21
22
23
24
25
26
27
28
  @Override
  public AbstractImageBundle createImageBundleAndOpen(int mode) throws IOException {
    Configuration conf = new Configuration();
    HipiImageBundle hib = new HipiImageBundle(new Path("/tmp/bundle.hib"), conf);
    hib.open(mode, true);
    return hib;
  }

  @Test
  public void testOffsets() throws IOException {
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
  @Test
  public void testAppend() throws IOException {
    //create image bundles
    Configuration conf = new Configuration();
    HipiImageBundle aib1 = new HipiImageBundle(new Path("/tmp/bundle1.hib"), conf);
    aib1.open(AbstractImageBundle.FILE_MODE_WRITE, true);   
    aib1.addImage(new FileInputStream("data/test/ImageBundleTestCase/read/0.jpg"), ImageType.JPEG_IMAGE);
    aib1.addImage(new FileInputStream("data/test/ImageBundleTestCase/read/1.jpg"), ImageType.JPEG_IMAGE);
    aib1.close();

    HipiImageBundle aib2 = new HipiImageBundle(new Path("/tmp/bundle2.hib"), conf);
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
    aib1.addImage(new FileInputStream("data/test/ImageBundleTestCase/read/0.jpg"), ImageType.JPEG_IMAGE);
    aib1.addImage(new FileInputStream("data/test/ImageBundleTestCase/read/1.jpg"), ImageType.JPEG_IMAGE);
    aib1.close();

    HipiImageBundle aib2 = new HipiImageBundle(new Path("/tmp/bundle2.hib"), conf);
    aib2.open(AbstractImageBundle.FILE_MODE_WRITE, true);
    aib2.addImage(new FileInputStream("data/test/ImageBundleTestCase/read/2.jpg"), ImageType.JPEG_IMAGE);
    aib2.addImage(new FileInputStream("data/test/ImageBundleTestCase/read/3.jpg"), ImageType.JPEG_IMAGE);
    aib2.close();

    HipiImageBundle aib1_in = new HipiImageBundle(new Path("/tmp/bundle1.hib"), conf);
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
    HipiImageBundle aib1_in = new HipiImageBundle(new Path("/tmp/bundle1.hib"), conf);
    HipiImageBundle aib2_in = new HipiImageBundle(new Path("/tmp/bundle2.hib"), conf);

    HipiImageBundle merged_hib = new HipiImageBundle(new Path("/tmp/merged_bundle.hib"), conf);
    merged_hib.open(HipiImageBundle.FILE_MODE_WRITE, true);
    merged_hib.append(aib1_in);
    merged_hib.append(aib2_in);
    merged_hib.close();
  }
}
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
    List<InputSplit> splits = new ArrayList<InputSplit>();
    for (FileStatus file : listStatus(job)) {
      Path path = file.getPath();
      FileSystem fs = path.getFileSystem(conf);
      HipiImageBundle hib = new HipiImageBundle(path, conf);
      hib.open(AbstractImageBundle.FILE_MODE_READ);
      // offset should be guaranteed to be in order
      List<Long> offsets = hib.getOffsets();
      BlockLocation[] blkLocations = fs.getFileBlockLocations(hib.getDataFile(), 0, offsets.get(offsets.size() - 1));
      if (numMapTasks == 0) {
        int i = 0, b = 0;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.