Package org.apache.blur.thrift.generated

Examples of org.apache.blur.thrift.generated.TableDescriptor

559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
      directory = longTermStorage;
    }

    ShardContext shardContext = ShardContext.create(tableContext, shard);

    TableDescriptor descriptor = tableContext.getDescriptor();
    boolean blockCacheEnabled = descriptor.isBlockCaching();
    if (blockCacheEnabled) {
      Set<String> blockCacheFileTypes = descriptor.getBlockCachingFileTypes();
      directory = _blockCacheDirectoryFactory.newDirectory(table, shard, directory, blockCacheFileTypes);
    }

    BlurIndex index = tableContext.newInstanceBlurIndex(shardContext, directory, _mergeScheduler, _searchExecutor,
        _indexCloser, _warmup);

    if (_clusterStatus.isReadOnly(true, _cluster, table)) {
      index = new BlurIndexReadOnly(index);
    }
    _filterCache.opening(table, shard, index);
    TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, _cluster, table);
    warmUp(index, tableDescriptor, shard);
    return index;
  }
View Full Code Here
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
  }

  private int getShardCount(String table) throws BlurException, TException {
    Integer numberOfShards = _tableShardCountMap.get(table);
    if (numberOfShards == null) {
      TableDescriptor descriptor = describe(table);
      numberOfShards = descriptor.shardCount;
      _tableShardCountMap.put(table, numberOfShards);
    }
    return numberOfShards;
  }
View Full Code Here
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import org.apache.hadoop.fs.Path;

public class CheckOutputSpecs {
 
  public static void checkOutputSpecs(Configuration config, int reducers) throws IOException, InterruptedException {
    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(config);
    if (tableDescriptor == null) {
      throw new IOException("setTableDescriptor needs to be called first.");
    }
    int shardCount = tableDescriptor.getShardCount();
    FileSystem fileSystem = BlurOutputFormat.getOutputPath(config).getFileSystem(config);
    Path tablePath = new Path(tableDescriptor.getTableUri());
    if (fileSystem.exists(tablePath)) {
      BlurUtil.validateShardCount(shardCount, fileSystem, tablePath);
    } else {
      throw new IOException("Table path [ " + tablePath + " ] doesn't exist for table [ " + tableDescriptor.getName()
          + " ].");
    }
    BlurUtil.validateWritableDirectory(fileSystem, tablePath);
    int reducerMultiplier = BlurOutputFormat.getReducerMultiplier(config);
    int validNumberOfReducers = reducerMultiplier * shardCount;
 
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
      return null;
    }
    ByteArrayInputStream inputStream = new ByteArrayInputStream(tableDesStr.getBytes());
    TIOStreamTransport transport = new TIOStreamTransport(inputStream);
    TJSONProtocol protocol = new TJSONProtocol(transport);
    TableDescriptor descriptor = new TableDescriptor();
    try {
      descriptor.read(protocol);
    } catch (TException e) {
      throw new IOException(e);
    }
    transport.close();
    return descriptor;
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
166
167
   * @param multiple
   *          the multiple to use.
   * @throws IOException
   */
  public static void setReducerMultiplier(Job job, int multiple) throws IOException {
    TableDescriptor tableDescriptor = getTableDescriptor(job.getConfiguration());
    if (tableDescriptor == null) {
      throw new IOException("setTableDescriptor needs to be called first.");
    }
    job.setNumReduceTasks(tableDescriptor.getShardCount() * multiple);
    Configuration configuration = job.getConfiguration();
    configuration.setInt(BLUR_OUTPUT_REDUCER_MULTIPLIER, multiple);
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
    LOG.info("Commiting Complete [{0}]", jobContext.getJobID());
  }

  private void makeSureNoEmptyShards(Configuration configuration, Path tableOutput) throws IOException {
    FileSystem fileSystem = tableOutput.getFileSystem(configuration);
    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
    int shardCount = tableDescriptor.getShardCount();
    for (int i = 0; i < shardCount; i++) {
      String shardName = BlurUtil.getShardName(i);
      fileSystem.mkdirs(new Path(tableOutput, shardName));
    }
  }
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
    String tableUri = new Path(TEST_ROOT_DIR + "/blur/" + tableName).toString();
    CsvBlurMapper.addColumns(job, "cf1", "col");

    TableDescriptor tableDescriptor = new TableDescriptor();
    tableDescriptor.setShardCount(1);
    tableDescriptor.setTableUri(tableUri);
    tableDescriptor.setName(tableName);

    Iface client = getClient();
    client.createTable(tableDescriptor);

    BlurOutputFormat.setupJob(job, tableDescriptor);
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
    };
    Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, "-c", "host:40010", "-d", "family1", "col1",
        "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", "file:///tmp/test1", "-i", "file:///tmp/test2");
    assertNotNull(job);
    Configuration configuration = job.getConfiguration();
    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
    assertEquals(tableDescriptor.getName(), "table1");
    Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
    assertEquals(2, inputs.size());
    Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
    assertEquals(2, familyAndColumnNameMap.size());
  }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
    Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, "-c", "host:40010", "-d", "family1", "col1",
        "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", "file:///tmp/test1", "-i", "file:///tmp/test2",
        "-S", "-C", "1000000", "2000000");
    assertNotNull(job);
    Configuration configuration = job.getConfiguration();
    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
    assertEquals(tableDescriptor.getName(), "table1");
    Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
    assertEquals(2, inputs.size());
    Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
    assertEquals(2, familyAndColumnNameMap.size());
  }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
    Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, "-c", "host:40010", "-d", "family1", "col1",
        "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", "file:///tmp/test1", "-i", "file:///tmp/test2",
        "-S", "-C", "1000000", "2000000", "-p", "SNAPPY");
    assertNotNull(job);
    Configuration configuration = job.getConfiguration();
    TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
    assertEquals(tableDescriptor.getName(), "table1");
    Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
    assertEquals(2, inputs.size());
    Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
    assertEquals(2, familyAndColumnNameMap.size());
    assertEquals("true", configuration.get(CsvBlurDriver.MAPRED_COMPRESS_MAP_OUTPUT));
View Full Code Here
TOP

Related Classes of org.apache.blur.thrift.generated.TableDescriptor

  • org.apache.blur.agent.collectors.blur.query.QueryCollectorTest
  • org.apache.blur.agent.collectors.blur.table.TableCollector
  • org.apache.blur.console.util.TableUtil
  • org.apache.blur.console.util.TableUtilTest
  • org.apache.blur.jdbc.BlurDatabaseMetaData
  • org.apache.blur.manager.AliasBlurFilterCacheTest
  • org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus
  • org.apache.blur.manager.clusterstatus.ZookeeperClusterStatusTest
  • org.apache.blur.manager.IndexManagerTest
  • org.apache.blur.manager.indexserver.AbstractDistributedIndexServer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.