Package org.apache.sqoop.model

Examples of org.apache.sqoop.model.MJob


    MConnection connection = getClient().newConnection("generic-jdbc-connector");
    fillConnectionForm(connection);
    createConnection(connection);

    // Job creation
    MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.EXPORT);

    // Connector values
    MFormList forms = job.getConnectorPart();
    forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName()));
    fillInputForm(job);
    createJob(job);

    runJob(job);
View Full Code Here


        // else the values will get set in the forms in the connector for
        // each connection.
        List<MForm> forms = newConnector.getJobForms(job.getType()).clone(false).getForms();
        MJobForms newJobForms = new MJobForms(job.getType(), forms);
        upgrader.upgrade(job.getConnectorPart(), newJobForms);
        MJob newJob = new MJob(connectorID, job.getConnectionId(),
          job.getType(), newJobForms, job.getFrameworkPart());
        newJob.setPersistenceId(job.getPersistenceId());

        // Transform form structures to objects for validations
        Object newConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(job.getType()));
        FormUtils.fromForms(newJob.getConnectorPart().getForms(), newConfigurationObject);

        Validation validation = validator.validateJob(newJob.getType(), newConfigurationObject);
        if (validation.getStatus().canProceed()) {
          updateJob(newJob, tx);
        } else {
          logInvalidModelObject("job", newJob, validation);
          upgradeSuccessful = false;
View Full Code Here

        // else the values will get set in the forms in the connector for
        // each connection.
        List<MForm> forms = framework.getJobForms(job.getType()).clone(false).getForms();
        MJobForms newJobForms = new MJobForms(job.getType(), forms);
        upgrader.upgrade(job.getFrameworkPart(), newJobForms);
        MJob newJob = new MJob(job.getConnectorId(), job.getConnectionId(),
          job.getType(), job.getConnectorPart(), newJobForms);
        newJob.setPersistenceId(job.getPersistenceId());

        // Transform form structures to objects for validations
        Object newConfigurationObject = ClassUtils.instantiate(FrameworkManager.getInstance().getJobConfigurationClass(job.getType()));
        FormUtils.fromForms(newJob.getFrameworkPart().getForms(), newConfigurationObject);

        Validation validation = validator.validateJob(newJob.getType(), newConfigurationObject);
        if (validation.getStatus().canProceed()) {
          updateJob(newJob, tx);
        } else {
          logInvalidModelObject("job", newJob, validation);
          upgradeSuccessful = false;
View Full Code Here

  private void cloneJob(Long jobId) throws IOException {
    printlnResource(Constants.RES_CLONE_CLONING_JOB, jobId);

    ConsoleReader reader = new ConsoleReader();

    MJob job = client.getJob(jobId);
    job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);

    ResourceBundle connectorBundle = client.getResourceBundle(job.getConnectorId());
    ResourceBundle frameworkBundle = client.getFrameworkResourceBundle();

    Status status = Status.FINE;

    // Remove persistent id as we're making a clone
    job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);

    printlnResource(Constants.RES_PROMPT_UPDATE_JOB_METADATA);
    do {
      // Print error introduction if needed
      if( !status.canProceed() ) {
        errorIntroduction();
      }

      // Fill in data from user
      if(!fillJob(reader, job, connectorBundle, frameworkBundle)) {
        return;
      }

      // Try to create
      status = client.createJob(job);
    } while(!status.canProceed());

    printlnResource(Constants.RES_CLONE_JOB_SUCCESSFUL, status.name(), job.getPersistenceId());
  }
View Full Code Here

      throw new SqoopException(ServerError.SERVER_0003,
        "Expected one job metadata but got " + jobs.size());
    }

    // Job object
    MJob job = jobs.get(0);

    // Verify that user is not trying to spoof us
    MJobForms connectorForms
      = ConnectorManager.getInstance().getConnectorMetadata(job.getConnectorId())
      .getJobForms(job.getType());
    MJobForms frameworkForms = FrameworkManager.getInstance().getFramework()
      .getJobForms(job.getType());

    if(!connectorForms.equals(job.getConnectorPart())
      || !frameworkForms.equals(job.getFrameworkPart())) {
      throw new SqoopException(ServerError.SERVER_0003,
        "Detected incorrect form structure");
    }

    // Responsible connector for this session
    SqoopConnector connector =
      ConnectorManager.getInstance().getConnector(job.getConnectorId());

    // Get validator objects
    Validator connectorValidator = connector.getValidator();
    Validator frameworkValidator = FrameworkManager.getInstance().getValidator();

    // We need translate forms to configuration objects
    Object connectorConfig = ClassUtils.instantiate(
      connector.getJobConfigurationClass(job.getType()));
    Object frameworkConfig = ClassUtils.instantiate(
      FrameworkManager.getInstance().getJobConfigurationClass(job.getType()));

    FormUtils.fromForms(job.getConnectorPart().getForms(), connectorConfig);
    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkConfig);

    // Validate both parts
    Validation connectorValidation =
      connectorValidator.validateJob(job.getType(), connectorConfig);
    Validation frameworkValidation =
      frameworkValidator.validateJob(job.getType(), frameworkConfig);

    Status finalStatus = Status.getWorstStatus(connectorValidation.getStatus(),
      frameworkValidation.getStatus());

    // Return back validations in all cases
    ValidationBean outputBean =
      new ValidationBean(connectorValidation, frameworkValidation);

    // If we're good enough let's perform the action
    if(finalStatus.canProceed()) {
      if(update) {
        AuditLoggerManager.getInstance()
            .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
            "update", "job", String.valueOf(job.getPersistenceId()));

        job.setLastUpdateUser(username);
        RepositoryManager.getInstance().getRepository().updateJob(job);
      } else {
        job.setCreationUser(username);
        job.setLastUpdateUser(username);
        RepositoryManager.getInstance().getRepository().createJob(job);
        outputBean.setId(job.getPersistenceId());

        AuditLoggerManager.getInstance()
            .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
            "create", "job", String.valueOf(job.getPersistenceId()));
      }

    }

    return outputBean;
View Full Code Here

        }
      }
    } else {
      long jid = Long.valueOf(sjid);

      MJob job = repository.findJob(jid);
      long connectorId = job.getConnectorId();

      bean = new JobBean(job);

      bean.addConnectorBundle(connectorId,
        ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
View Full Code Here

    if (entity instanceof MConnection) {
      MConnection connection = (MConnection) entity;
      formList.addAll(connection.getConnectorPart().getForms());
      formList.addAll(connection.getFrameworkPart().getForms());
    } else if(entity instanceof MJob) {
      MJob job = (MJob) entity;
      formList.addAll(job.getConnectorPart().getForms());
      formList.addAll(job.getFrameworkPart().getForms());
    }
    for(MForm form : formList) {
      if(form.getValidationStatus() == Status.ACCEPTABLE) {
        if(showMessage) {
          print("\n@|yellow %s|@\n", resourceString(Constants.RES_FORMDISPLAYER_FORM_WARNING));
View Full Code Here

  public MSubmission submit(long jobId, HttpEventContext ctx) {
    String username = ctx.getUsername();

    Repository repository = RepositoryManager.getInstance().getRepository();

    MJob job = repository.findJob(jobId);
    if (job == null) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0004,
        "Unknown job id " + jobId);
    }

    if (!job.getEnabled()) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0009,
        "Job id: " + job.getPersistenceId());
    }

    MConnection connection = repository.findConnection(job.getConnectionId());

    if (!connection.getEnabled()) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0010,
        "Connection id: " + connection.getPersistenceId());
    }

    SqoopConnector connector =
      ConnectorManager.getInstance().getConnector(job.getConnectorId());

    // Transform forms to connector specific classes
    Object connectorConnection = ClassUtils.instantiate(
      connector.getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getConnectorPart().getForms(),
      connectorConnection);

    Object connectorJob = ClassUtils.instantiate(
      connector.getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getConnectorPart().getForms(), connectorJob);

    // Transform framework specific forms
    Object frameworkConnection = ClassUtils.instantiate(
      FrameworkManager.getInstance().getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getFrameworkPart().getForms(),
      frameworkConnection);

    Object frameworkJob = ClassUtils.instantiate(
      FrameworkManager.getInstance().getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);

    // Create request object
    MSubmission summary = new MSubmission(jobId);
    SubmissionRequest request = executionEngine.createSubmissionRequest();

    summary.setCreationUser(username);
    summary.setLastUpdateUser(username);

    // Save important variables to the submission request
    request.setSummary(summary);
    request.setConnector(connector);
    request.setConfigConnectorConnection(connectorConnection);
    request.setConfigConnectorJob(connectorJob);
    request.setConfigFrameworkConnection(frameworkConnection);
    request.setConfigFrameworkJob(frameworkJob);
    request.setJobType(job.getType());
    request.setJobName(job.getName());
    request.setJobId(job.getPersistenceId());
    request.setNotificationUrl(notificationBaseUrl + jobId);

    // Let's register all important jars
    // sqoop-common
    request.addJarForClass(MapContext.class);
    // sqoop-core
    request.addJarForClass(FrameworkManager.class);
    // sqoop-spi
    request.addJarForClass(SqoopConnector.class);
    // Execution engine jar
    request.addJarForClass(executionEngine.getClass());
    // Connector in use
    request.addJarForClass(connector.getClass());

    // Extra libraries that Sqoop code requires
    request.addJarForClass(JSONValue.class);

    // Get connector callbacks
    switch (job.getType()) {
      case IMPORT:
        request.setConnectorCallbacks(connector.getImporter());
        break;
      case EXPORT:
        request.setConnectorCallbacks(connector.getExporter());
        break;
      default:
        throw new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }
    LOG.debug("Using callbacks: " + request.getConnectorCallbacks());

    // Initialize submission from connector perspective
    CallbackBase baseCallbacks = request.getConnectorCallbacks();

    Class<? extends Initializer> initializerClass = baseCallbacks
      .getInitializer();
    Initializer initializer = (Initializer) ClassUtils
      .instantiate(initializerClass);

    if (initializer == null) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0006,
        "Can't create initializer instance: " + initializerClass.getName());
    }

    // Initializer context
    InitializerContext initializerContext = new InitializerContext(
      request.getConnectorContext());

    // Initialize submission from connector perspective
    initializer.initialize(initializerContext,
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob());

    // Add job specific jars to
    request.addJars(initializer.getJars(initializerContext,
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob()));

    // Retrieve and persist the schema
    request.getSummary().setConnectorSchema(initializer.getSchema(
      initializerContext,
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob()
      ));

    // Bootstrap job from framework perspective
    switch (job.getType()) {
      case IMPORT:
        prepareImportSubmission(request);
        break;
      case EXPORT:
        prepareExportSubmission(request);
        break;
      default:
        throw new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }

    // Make sure that this job id is not currently running and submit the job
    // only if it's not.
    synchronized (getClass()) {
View Full Code Here

    }

    // Load prepared connections into database
    loadJobs();

    MJob jobImport = handler.findJob(1, getDerbyConnection());
    assertNotNull(jobImport);
    assertEquals(1, jobImport.getPersistenceId());
    assertEquals("JA", jobImport.getName());
    assertEquals(MJob.Type.IMPORT, jobImport.getType());

    List<MForm> forms;

    // Check connector part
    forms = jobImport.getConnectorPart().getForms();
    assertEquals("Value5", forms.get(0).getInputs().get(0).getValue());
    assertNull(forms.get(0).getInputs().get(1).getValue());
    assertEquals("Value7", forms.get(1).getInputs().get(0).getValue());
    assertNull(forms.get(1).getInputs().get(1).getValue());

    // Check framework part
    forms = jobImport.getFrameworkPart().getForms();
    assertEquals("Value17", forms.get(0).getInputs().get(0).getValue());
    assertNull(forms.get(0).getInputs().get(1).getValue());
    assertEquals("Value19", forms.get(1).getInputs().get(0).getValue());
    assertNull(forms.get(1).getInputs().get(1).getValue());
  }
View Full Code Here

    assertFalse(handler.inUseJob(3, getDerbyConnection()));
    assertFalse(handler.inUseJob(4, getDerbyConnection()));
  }

  public void testCreateJob() throws Exception {
    MJob job = getJob();

    // Load some data
    fillJob(job);

    handler.createJob(job, getDerbyConnection());

    assertEquals(1, job.getPersistenceId());
    assertCountForTable("SQOOP.SQ_JOB", 1);
    assertCountForTable("SQOOP.SQ_JOB_INPUT", 4);

    MJob retrieved = handler.findJob(1, getDerbyConnection());
    assertEquals(1, retrieved.getPersistenceId());

    List<MForm> forms;
    forms = job.getConnectorPart().getForms();
    assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
    assertNull(forms.get(0).getInputs().get(1).getValue());
View Full Code Here

TOP

Related Classes of org.apache.sqoop.model.MJob

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.