Examples of org.apache.hadoop.conf.Configuration.writeXml()

Class org.apache.hadoop.conf.Configuration

Examples of org.apache.hadoop.conf.Configuration.writeXml()

org.apache.hadoop.conf.Configuration.writeXml()
Write out the non-default properties in this configuration to the given {@link OutputStream}. @param out the output stream to write to.

      LOG.error("Problem creating temporary hive-site.xml conf file at {}", newHiveConfFile, e);
      throw Throwables.propagate(e);
    }


    try {
      conf.writeXml(fos);
    } catch (IOException e) {
      LOG.error("Could not write modified configuration to temporary hive-site.xml at {}", newHiveConfFile, e);
      throw Throwables.propagate(e);
    } finally {
      Closeables.closeQuietly(fos);

View Full Code Here

            m_fileSys.mkdirs(m_fileSys.getWorkingDirectory());


            // Write core-site.xml
            Configuration core_site = new Configuration(false);
            core_site.set(FileSystem.FS_DEFAULT_NAME_KEY, m_dfs_conf.get(FileSystem.FS_DEFAULT_NAME_KEY));
            core_site.writeXml(new FileOutputStream(CORE_CONF_FILE));


            Configuration hdfs_site = new Configuration(false);
            for (Entry<String, String> conf : m_dfs_conf) {
                if (ArrayUtils.contains(m_dfs_conf.getPropertySources(conf.getKey()), "programatically")) {
                    hdfs_site.set(conf.getKey(), m_dfs_conf.getRaw(conf.getKey()));

View Full Code Here

            for (Entry<String, String> conf : m_dfs_conf) {
                if (ArrayUtils.contains(m_dfs_conf.getPropertySources(conf.getKey()), "programatically")) {
                    hdfs_site.set(conf.getKey(), m_dfs_conf.getRaw(conf.getKey()));
                }
            }
            hdfs_site.writeXml(new FileOutputStream(HDFS_CONF_FILE));


            // Build mini YARN cluster
            m_mr = new MiniMRYarnCluster("PigMiniCluster", 2);
            m_mr.init(m_dfs_conf);
            m_mr.start();

View Full Code Here

                    }
                }
            }


            mapred_site.writeXml(new FileOutputStream(MAPRED_CONF_FILE));
            yarn_site.writeXml(new FileOutputStream(YARN_CONF_FILE));


            // Write tez-site.xml
            Configuration tez_conf = new Configuration(false);
            // TODO PIG-3659 - Remove this once memory management is fixed
            tez_conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, "20");

View Full Code Here

            // Set to a lower value so that tests don't get stuck for long because of 1 AM running at a time
            tez_conf.set(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, "20");
            // Lower the max task attempts to 2 so that negative tests fail
            // faster. By default, tasks retry 4 times
            tez_conf.set(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, "2");
            tez_conf.writeXml(new FileOutputStream(TEZ_CONF_FILE));


            // Copy tez jars to hdfs
            m_fileSys.mkdirs(new Path("/tez/lib"));
            FileFilter fileFilter = new RegexFileFilter("tez-.+\\.jar$");
            File[] tezJars = TEZ_LIB_DIR.listFiles(fileFilter);

View Full Code Here

        // (.staging/jobid/files - RecoveryService will need to be patched)
        FSDataOutputStream jobFileOut = null;
        try {
          if (logDirConfPath != null) {
            jobFileOut = stagingDirFS.create(logDirConfPath, true);
            conf.writeXml(jobFileOut);
            jobFileOut.close();
          }
        } catch (IOException e) {
          LOG.info("Failed to write the job configuration file", e);
          throw e;

View Full Code Here


    //we do this trick because the MR AppMaster is started in another VM and
    //the HttpServer configuration is not loaded from the job.xml but from the
    //site.xml files in the classpath
    Writer writer = new FileWriter(new File(classpathDir, CONFIG_SITE_XML));
    conf.writeXml(writer);
    writer.close();


    conf.setInt(HttpServer.HTTP_MAX_THREADS, 10);
    conf.addResource(CONFIG_SITE_XML);
    server = createServer("test", conf);

View Full Code Here

      Configuration yarnClusterConfig = yarnCluster.getConfig();
      yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent());
      //write the document to a buffer (not directly to the file, as that
      //can cause the file being written to get read -which will then fail.
      ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
      yarnClusterConfig.writeXml(bytesOut);
      bytesOut.close();
      //write the bytes to the file in the classpath
      OutputStream os = new FileOutputStream(new File(url.getPath()));
      os.write(bytesOut.toByteArray());
      os.close();

View Full Code Here

        confVarFile.deleteOnExit();


        applyDefaultNonNullConfVars(conf);


        FileOutputStream fout = new FileOutputStream(confVarFile);
        conf.writeXml(fout);
        fout.close();
        confVarURL = confVarFile.toURI().toURL();
      } catch (Exception e) {
        // We're pretty screwed if we can't load the default conf vars
        throw new RuntimeException("Failed to initialize default Hive configuration variables!", e);

View Full Code Here

        // (.staging/jobid/files - RecoveryService will need to be patched)
        FSDataOutputStream jobFileOut = null;
        try {
          if (logDirConfPath != null) {
            jobFileOut = stagingDirFS.create(logDirConfPath, true);
            conf.writeXml(jobFileOut);
            jobFileOut.close();
          }
        } catch (IOException e) {
          LOG.info("Failed to write the job configuration file", e);
          throw e;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.