Examples of org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles$LoadQueueItem

Package org.apache.hadoop.hbase.mapreduce

Examples of org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles$LoadQueueItem

org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles
Represents an HFile waiting to be loaded. An queue is used in this class in order to support the case where a region has split during the process of the load. When this happens, the HFile is split into two physical parts across the new region boundary, and each part is added back into the queue. The import process finishes when the queue is empty.

      setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx"));


      HTable table = new HTable(conf, tableName);
      try {
        TEST_UTIL.waitTableAvailable(tableName, 30000);
        LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
        loader.doBulkLoad(loadPath, table);
      } finally {
        table.close();
      }
    }

View Full Code Here

    Path familyDir = new Path(dir, Bytes.toString(A));


    createHFile(util.getConfiguration(), fs, new Path(familyDir,Bytes.toString(A)), A, A);


    //Bulk load
    new LoadIncrementalHFiles(conf).doBulkLoad(dir, new HTable(conf, tableName));


    verifyMethodResult(SimpleRegionObserver.class,
        new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
        tableName,
        new Boolean[] {true, true}

View Full Code Here


      HTable table = new HTable(conf, tableName);
      try {
        TEST_UTIL.waitTableAvailable(tableName, 30000);
        conf.setBoolean("hbase.mapreduce.bulkload.assign.sequenceNumbers", true);
        LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
        loader.doBulkLoad(loadPath, table);
      } finally {
        table.close();
      }
    }

View Full Code Here


      HTable table = new HTable(conf, tableName);
      try {
        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
        TEST_UTIL.waitTableEnabled(admin, tableName.getName());
        LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
        loader.doBulkLoad(loadPath, table);
      } finally {
        table.close();
      }
    }

View Full Code Here

    Path familyDir = new Path(dir, Bytes.toString(A));


    createHFile(util.getConfiguration(), fs, new Path(familyDir,Bytes.toString(A)), A, A);


    //Bulk load
    new LoadIncrementalHFiles(conf).doBulkLoad(dir, new HTable(conf, tableName));


    verifyMethodResult(SimpleRegionObserver.class,
        new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
        tableName,
        new Boolean[] {true, true}

View Full Code Here

    Path familyDir = new Path(dir, Bytes.toString(A));


    createHFile(util.getConfiguration(), fs, new Path(familyDir,Bytes.toString(A)), A, A);


    //Bulk load
    new LoadIncrementalHFiles(conf).doBulkLoad(dir, new HTable(conf, tableName));


    verifyMethodResult(SimpleRegionObserver.class,
        new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
        tableName,
        new Boolean[] {true, true}

View Full Code Here

        outputPath);


    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    new LoadIncrementalHFiles(HBASE_TEST_UTILITY.getConfiguration())
        .doBulkLoad(outputPath, testTable);


    Map<String, Long> EXPECTED = ImmutableMap.<String, Long>builder()
        .put("__EMPTY__", 1470L)
        .put("the", 620L)

View Full Code Here

    Path inputPath = copyResourceFileToHDFS("shakes.txt");
    Path outputPath1 = getTempPathOnHDFS("out1");
    Path outputPath2 = getTempPathOnHDFS("out2");
    HTable table1 = createTable(26);
    HTable table2 = createTable(26);
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(HBASE_TEST_UTILITY.getConfiguration());


    PCollection<String> shakespeare = pipeline.read(At.textFile(inputPath, Writables.strings()));
    PCollection<String> words = split(shakespeare, "\\s+");
    PCollection<String> shortWords = words.filter(SHORT_WORD_FILTER);
    PCollection<String> longWords = words.filter(FilterFns.not(SHORT_WORD_FILTER));
    PTable<String, Long> shortWordCounts = shortWords.count();
    PTable<String, Long> longWordCounts = longWords.count();
    HFileUtils.writePutsToHFilesForIncrementalLoad(
        convertToPuts(shortWordCounts),
        table1,
        outputPath1);
    HFileUtils.writePutsToHFilesForIncrementalLoad(
        convertToPuts(longWordCounts),
        table2,
        outputPath2);


    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    loader.doBulkLoad(outputPath1, table1);
    loader.doBulkLoad(outputPath2, table2);


    assertEquals(396L, getWordCountFromTable(table1, "of"));
    assertEquals(427L, getWordCountFromTable(table2, "and"));
  }

View Full Code Here

          try {
            baseOutputCommitter.commitJob(jobContext);
            Configuration conf = jobContext.getConfiguration();
            try {
              //import hfiles
              new LoadIncrementalHFiles(conf)
                .doBulkLoad(HFileOutputFormat.getOutputPath(jobContext),
                  new HTable(conf,
                    conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY)));
            } catch (Exception e) {
              throw new IOException("BulkLoad failed.", e);

View Full Code Here

        outputPath);


    PipelineResult result = pipeline.run();
    assertTrue(result.succeeded());


    new LoadIncrementalHFiles(HBASE_TEST_UTILITY.getConfiguration())
        .doBulkLoad(outputPath, testTable);


    Map<String, Long> EXPECTED = ImmutableMap.<String, Long>builder()
        .put("__EMPTY__", 1470L)
        .put("the", 620L)

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles$LoadQueueItem

com.google.common.util.concurrent.ThreadFactoryBuilder

com.lightboxtechnologies.spectrum.ExtractData

com.salesforce.phoenix.map.reduce.CSVBulkLoader

com.splunk.shuttl.integration.hadoop.hbase.JobRunner

com.urbanairship.datacube.backfill.HBaseSnapshotter

org.apache.bigtop.itest.hbase.smoke.TestHFileOutputFormat

org.apache.bigtop.itest.hbase.smoke.TestLoadIncrementalHFiles

org.apache.commons.lang.mutable.MutableInt

org.apache.crunch.io.hbase.HFileTargetIT

org.apache.hadoop.conf.Configuration

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.