Package org.kiji.schema.testutil

Source Code of org.kiji.schema.testutil.IntegrationHelper

/**
* (c) Copyright 2012 WibiData, Inc.
*
* See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.kiji.schema.testutil;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Arrays;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.kiji.schema.Kiji;
import org.kiji.schema.KijiInstaller;
import org.kiji.schema.KijiURI;
import org.kiji.schema.layout.KijiTableLayout;
import org.kiji.schema.layout.KijiTableLayouts;
import org.kiji.schema.tools.BaseTool;
import org.kiji.schema.tools.KijiToolLauncher;
import org.kiji.schema.util.InstanceBuilder;
import org.kiji.schema.util.ResourceUtils;

/**
* IntegrationHelper provides methods for installing and managing a Kiji instance during a test.
*
* <p>If you would like to install a new Kiji instance, it will randomly generate a new
* Kiji instance name so you don't step on the toes of other tests.  If you do create Kiji
* instances, you uninstall them when you are finished.</p>
*
* <p>Almost by definition, any test that uses this will be prefixed with
* "IntegrationTest" and will thus be run in 'mvn integration-test' instead of a nice fast
* 'mvn test'.</p>
*
* <p>If you only need one Kiji instance during your test, you can extend {@link
* org.kiji.schema.testutil.AbstractKijiIntegrationTest}.</p>
*/
public class IntegrationHelper extends Configured {
  private static final Logger LOG = LoggerFactory.getLogger(IntegrationHelper.class);

  /** Some test data that can be loaded into the "foo" table. */
  public static final String INPUT_FILE = "org/kiji/schema/test-data.csv";

  /** A bulk import input-format file describing the test-data file above. */
  private static final String INPUT_FORMAT_FILE
      = "org/kiji/schema/inputlayout/test-integration-helper-input.txt";

  /** A directory within the DFS where tests can create files. */
  private static final String DFS_TEST_DIR = "test-data";

  /**
   * Creates a new <code>IntegrationHelper</code> instance.
   *
   * @param conf A configuration specifying the HBase cluster.
   */
  public IntegrationHelper(Configuration conf) {
    super(conf);
  }

  /**
   * Copies a local file into the default filesystem (which is HDFS if you've started an
   * HBase cluster).
   *
   * @param localFile The file to copy.
   * @param destPath A relative destination path to be used within the shared tmp dir.
   * @return The path of the file in HDFS.
   */
  public Path copyToDfs(File localFile, String destPath) throws IOException {
    Path target = getDfsPath(destPath);
    FileSystem fs = FileSystem.get(getConf());
    if (!fs.exists(target)) {
      // Only copy if it doesn't already exist.
      FileUtil.copy(localFile, fs, target, false, getConf());
    }
    return target;
  }

  /**
   * Get the DFS path corresponding with a local file path.
   *
   * @param destPath A relative destination path to be used within the shared tmp dir.
   * @return The path to the file in the mini HDFS filesystem.
   * @throws java.io.IOException If there is an error.
   */
  public Path getDfsPath(String destPath) throws IOException {
    return new Path(DFS_TEST_DIR, destPath).makeQualified(FileSystem.get(getConf()));
  }

  /**
   * Installs Kiji metadata tables in the HBase instance.
   *
   * @param kijiURI The uri of the Kiji instance to install.
   * @throws Exception If there is an error.
   */
  public void installKiji(KijiURI kijiURI) throws Exception {
    KijiInstaller.get().install(kijiURI, getConf());
  }

  /**
   * Uninstall a Kiji instance.
   *
   * @throws Exception If there is an error.
   */
  public void uninstallKiji(KijiURI kijiURI) throws Exception {
    KijiInstaller.get().uninstall(kijiURI, getConf());
  }

  /**
   * Runs a tool and captures the console output.
   *
   * @param conf The configuration to run the tool with.
   * @param tool The tool to run.
   * @param args The command-line args to pass to the tool.
   * @return A result with the captured tool output.
   * @throws Exception If there is an error.
   */
  public ToolResult runTool(Configuration conf, BaseTool tool, String[] args) throws Exception {
    // Capture STDOUT to an in-memory output stream.
    ByteArrayOutputStream output = new ByteArrayOutputStream();
    tool.setPrintStream(new PrintStream(output));

    // Run the tool.
    LOG.debug("Running tool " + tool.getClass().getName() + " with args " + Arrays.toString(args));
    KijiToolLauncher launcher = new KijiToolLauncher();
    launcher.setConf(conf);
    int exitCode = launcher.run(tool, args);
    return new ToolResult(exitCode, output);
  }

  /**
   * Writes the contents of the input stream to the temporary file provided.
   *
   * @param tempFile The temporary file.
   * @param inStream The input stream.
   */
  private static void writeTempFile(File tempFile, InputStream inStream) throws IOException {
    OutputStream outStream = new FileOutputStream(tempFile);
    try {
      IOUtils.copy(inStream, outStream);
    } finally {
      if (null != outStream) {
        ResourceUtils.closeOrLog(outStream);
      }
    }
  }

  /**
   * Writes the files necessary to create the foo table.
   *
   * @param dataFile A file to hold the table data.
   * @param formatFile A file to hold the data format.
   * @throws IOException If there is an error while writing the files.
   */
  private void writeFooTableFiles(File dataFile, File formatFile) throws IOException {
    InputStream dataStream = null;
    InputStream formatStream = null;
    try {
      dataStream = getClass().getClassLoader().getResourceAsStream(INPUT_FILE);
      formatStream = getClass().getClassLoader().getResourceAsStream(INPUT_FORMAT_FILE);

      writeTempFile(dataFile, dataStream);
      writeTempFile(formatFile, formatStream);
    } finally {
      if (null != dataStream) {
        ResourceUtils.closeOrLog(dataStream);
      }
      if (null != formatStream) {
        ResourceUtils.closeOrLog(formatStream);
      }
    }
  }

  /**
   * Creates and populates a test table of users called 'foo'.
   *
   * @param kijiURI The KijiURI to create the table in.
   * @throws Exception If there is an error.
   */
  public void createAndPopulateFooTable(KijiURI kijiURI) throws Exception {
    final Kiji kiji = Kiji.Factory.open(kijiURI, getConf());
    try {
      final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
      final long timestamp = System.currentTimeMillis();

      new InstanceBuilder(kiji)
          .withTable(layout.getName(), layout)
              .withRow("gwu@usermail.example.com")
                  .withFamily("info")
                      .withQualifier("email").withValue(timestamp, "gwu@usermail.example.com")
                      .withQualifier("name").withValue(timestamp, "Garrett Wu")
              .withRow("aaron@usermail.example.com")
                  .withFamily("info")
                      .withQualifier("email").withValue(timestamp, "aaron@usermail.example.com")
                      .withQualifier("name").withValue(timestamp, "Aaron Kimball")
              .withRow("christophe@usermail.example.com")
                  .withFamily("info")
                      .withQualifier("email")
                          .withValue(timestamp, "christophe@usermail.example.com")
                      .withQualifier("name").withValue(timestamp, "Christophe Bisciglia")
              .withRow("kiyan@usermail.example.com")
                  .withFamily("info")
                      .withQualifier("email").withValue(timestamp, "kiyan@usermail.example.com")
                      .withQualifier("name").withValue(timestamp, "Kiyan Ahmadizadeh")
              .withRow("john.doe@gmail.com")
                  .withFamily("info")
                      .withQualifier("email").withValue(timestamp, "john.doe@gmail.com")
                      .withQualifier("name").withValue(timestamp, "John Doe")
              .withRow("jane.doe@gmail.com")
                  .withFamily("info")
                      .withQualifier("email").withValue(timestamp, "jane.doe@gmail.com")
                      .withQualifier("name").withValue(timestamp, "Jane Doe")
          .build();

    } finally {
      kiji.release();
    }

    // Create the temp files needed to populate the foo table.
    final File dataFile = File.createTempFile("data", ".csv");
    dataFile.deleteOnExit();
    final File dataFormatFile = File.createTempFile("data-format", ".csv");
    dataFormatFile.deleteOnExit();
    // Write the temp files needed for the test.
    writeFooTableFiles(dataFile, dataFormatFile);
    dataFile.delete();
    dataFormatFile.delete();
  }

  /**
   * Deletes the table created with createAndPopulateFooTable().
   *
   * @param kijiURI The KijiURI to delete the table from.
   * @throws Exception If there is an error.
   */
  public void deleteFooTable(KijiURI kijiURI) throws Exception {
    final Kiji kiji = Kiji.Factory.open(kijiURI, getConf());
    try {
      kiji.deleteTable("foo");
    } finally {
      kiji.release();
    }
  }
}
TOP

Related Classes of org.kiji.schema.testutil.IntegrationHelper

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.