Package org.apache.accumulo.core.conf

Examples of org.apache.accumulo.core.conf.AccumuloConfiguration

3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
        Set<FileRef> smallestFiles = removeSmallest(filesToCompact, numToCompact);

        FileRef fileName = getNextMapFilename((filesToCompact.size() == 0 && !propogateDeletes) ? "A" : "C");
        FileRef compactTmpName = new FileRef(fileName.path().toString() + "_tmp");

        AccumuloConfiguration tableConf = createTableConfiguration(acuTableConf, plan);

        Span span = Trace.start("compactFiles");
        try {

          CompactionEnv cenv = new CompactionEnv() {
View Full Code Here
858
859
860
861
862
863
864
865
866
867
868
    String even = new File(importDir, "even.rf").toString();
    String odd = new File(importDir, "odd.rf").toString();
    File errorsDir = new File(getFolder(), "errors");
    errorsDir.mkdir();
    fs.mkdirs(new Path(errorsDir.toString()));
    AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
    FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf);
    evenWriter.startDefaultLocalityGroup();
    FileSKVWriter oddWriter = FileOperations.getInstance().openWriter(odd, fs, conf, aconf);
    oddWriter.startDefaultLocalityGroup();
    long timestamp = System.currentTimeMillis();
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    conn.tableOperations().create(table);
    insertData(table, currentTimeMillis());

    ClientConfiguration clientConf = new ClientConfiguration().withInstance(conn.getInstance().getInstanceName()).withZkHosts(
        conn.getInstance().getZooKeepers());
    AccumuloConfiguration clusterClientConf = conn.getInstance().getConfiguration();

    // Pass SSL and CredentialProvider options into the ClientConfiguration given to AccumuloInputFormat
    boolean sslEnabled = Boolean.valueOf(clusterClientConf.get(Property.INSTANCE_RPC_SSL_ENABLED));
    if (sslEnabled) {
      ClientProperty[] sslProperties = new ClientProperty[] {ClientProperty.INSTANCE_RPC_SSL_ENABLED, ClientProperty.INSTANCE_RPC_SSL_CLIENT_AUTH,
          ClientProperty.RPC_SSL_KEYSTORE_PATH, ClientProperty.RPC_SSL_KEYSTORE_TYPE, ClientProperty.RPC_SSL_KEYSTORE_PASSWORD,
          ClientProperty.RPC_SSL_TRUSTSTORE_PATH, ClientProperty.RPC_SSL_TRUSTSTORE_TYPE, ClientProperty.RPC_SSL_TRUSTSTORE_PASSWORD,
          ClientProperty.RPC_USE_JSSE, ClientProperty.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS};

      for (ClientProperty prop : sslProperties) {
        // The default property is returned if it's not in the ClientConfiguration so we don't have to check if the value is actually defined
        clientConf.setProperty(prop, clusterClientConf.get(prop.getAccumuloProperty()));
      }
    }

    @SuppressWarnings("deprecation")
    Job job = new Job();
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
    public AccumuloConfiguration getConfiguration() {
      if (siteFile == null) {
        return SiteConfiguration.getInstance(DefaultConfiguration.getInstance());
      } else {
        return new AccumuloConfiguration() {
          Configuration xml = new Configuration();
          {
            xml.addResource(new Path(siteFile));
          }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
 
  @Override
  public RecordWriter<Key,Value> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
    // get the path of the temporary output file
    final Configuration conf = job;
    final AccumuloConfiguration acuConf = getAccumuloConfiguration(job);
   
    final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
    final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension);
   
    final LRUMap validVisibilities = new LRUMap(ConfiguratorBase.getVisibilityCacheSize(conf));
   
    return new RecordWriter<Key,Value>() {
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    AccumuloFileOutputFormat.setFileBlockSize(job, b);
    AccumuloFileOutputFormat.setDataBlockSize(job, c);
    AccumuloFileOutputFormat.setIndexBlockSize(job, d);
    AccumuloFileOutputFormat.setCompressionType(job, e);

    AccumuloConfiguration acuconf = AccumuloFileOutputFormat.getAccumuloConfiguration(job);

    assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
    assertEquals(300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
    assertEquals(50l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
    assertEquals(10l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
    assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));

    a = 17;
    b = 1300l;
    c = 150l;
    d = 110l;
    e = "lzo";

    job = new JobConf();
    AccumuloFileOutputFormat.setReplication(job, a);
    AccumuloFileOutputFormat.setFileBlockSize(job, b);
    AccumuloFileOutputFormat.setDataBlockSize(job, c);
    AccumuloFileOutputFormat.setIndexBlockSize(job, d);
    AccumuloFileOutputFormat.setCompressionType(job, e);

    acuconf = AccumuloFileOutputFormat.getAccumuloConfiguration(job);

    assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
    assertEquals(1300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
    assertEquals(150l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
    assertEquals(110l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
    assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));

  }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
  }
 
  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    AccumuloConfiguration aconf = SiteConfiguration.getInstance(DefaultConfiguration.getInstance());
    // TODO ACCUMULO-2462 This will only work for RFiles (path only, not URI) in HDFS when the correct filesystem for the given file
    // is on Property.INSTANCE_DFS_DIR or, when INSTANCE_DFS_DIR is not defined, is on the default filesystem
    // defined in the Hadoop's core-site.xml
    //
    // A workaround is to always provide a URI to this class
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
      if (!conn.tableOperations().exists(opts.tableName)) {
        System.err.println("table " + opts.tableName + " does not exist");
        return;
      }
      if (opts.goalSize == null || opts.goalSize < 1) {
        AccumuloConfiguration tableConfig = new ConfigurationCopy(conn.tableOperations().getProperties(opts.tableName));
        opts.goalSize = tableConfig.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD);
      }
     
      message("Merging tablets in table %s to %d bytes", opts.tableName, opts.goalSize);
      mergomatic(conn, opts.tableName, opts.begin, opts.end, opts.goalSize, opts.force);
    } catch (Exception ex) {
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    }
  }
 
  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    AccumuloConfiguration siteConf = SiteConfiguration.getInstance(DefaultConfiguration.getInstance());
    // TODO ACCUMULO-2462 not going to operate as expected with volumes when a path, not URI, is given
    FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, siteConf).getFileSystem();
    FileSystem localFs = FileSystem.getLocal(conf);
    Path path = new Path(args[0]);
    FileSystem fs;
View Full Code Here
112
113
114
115
116
117
118
119
  public static Pair<String,ClientService.Client> getConnection(Instance instance) throws TTransportException {
    return getConnection(instance, true);
  }
 
  public static Pair<String,ClientService.Client> getConnection(Instance instance, boolean preferCachedConnections) throws TTransportException {
    AccumuloConfiguration conf = ServerConfigurationUtil.getConfiguration(instance);
    return getConnection(instance, preferCachedConnections, conf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
  }
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.conf.AccumuloConfiguration

  • org.apache.accumulo.core.cli.ClientOpts
  • org.apache.accumulo.core.client.admin.TableOperationsImpl
  • org.apache.accumulo.core.client.impl.OfflineIterator
  • org.apache.accumulo.core.client.impl.ServerClient
  • org.apache.accumulo.core.client.impl.ServerConfigurationUtil
  • org.apache.accumulo.core.client.impl.ServerConfigurationUtilTest
  • org.apache.accumulo.core.client.impl.TableOperationsImpl
  • org.apache.accumulo.core.client.mapred.AccumuloFileOutputFormat
  • org.apache.accumulo.core.client.mapred.AccumuloFileOutputFormatTest
  • org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.