Package io.druid.timeline

Examples of io.druid.timeline.DataSegment

108
109
110
111
112
113
114
115
116
117
118
119
120
    try {
      FileSystem fs = descriptorInfoDir.getFileSystem(conf);

      for (FileStatus status : fs.listStatus(descriptorInfoDir)) {
        final DataSegment segment = jsonMapper.readValue(fs.open(status.getPath()), DataSegment.class);
        publishedSegmentsBuilder.add(segment);
        log.info("Adding segment %s to the list of published segments", segment.getIdentifier());
      }
    }
    catch (IOException e) {
      throw Throwables.propagate(e);
    }
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
145
  @Test
  public void testSingleAnnounce() throws Exception
  {
    Iterator<DataSegment> segIter = testSegments.iterator();
    DataSegment firstSegment = segIter.next();
    DataSegment secondSegment = segIter.next();

    segmentAnnouncer.announceSegment(firstSegment);

    List<String> zNodes = cf.getChildren().forPath(testSegmentsPath);
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
  {
    return new TestMergeTask(
        taskId,
        "dummyDs",
        Lists.<DataSegment>newArrayList(
            new DataSegment(
                "dummyDs",
                new Interval(new DateTime(), new DateTime().plus(1)),
                new DateTime().toString(),
                null,
                null,
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    List<DataSegment> cachedSegments = Lists.newArrayList();
    for (File file : baseDir.listFiles()) {
      log.info("Loading segment cache file [%s]", file);
      try {
        DataSegment segment = jsonMapper.readValue(file, DataSegment.class);
        if (serverManager.isSegmentCached(segment)) {
          cachedSegments.add(segment);
        } else {
          log.warn("Unable to find cache file for %s. Deleting lookup entry", segment.getIdentifier());

          File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier());
          if (!segmentInfoCacheFile.delete()) {
            log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile);
          }
        }
      }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
      return;
    }

    final String dataSource = segmentsIter.next().getDataSource();
    while (segmentsIter.hasNext()) {
      DataSegment next = segmentsIter.next();
      if (!dataSource.equals(next.getDataSource())) {
        throw new IAE("Cannot merge segments of different dataSources[%s] and [%s]", dataSource, next.getDataSource());
      }
    }

    runQuery(new ClientAppendQuery(dataSource, segments));
  }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  )
  {
    // Verify that these segments are all in the same partition set

    Preconditions.checkArgument(!segments.isEmpty(), "segments nonempty");
    final DataSegment firstSegment = segments.iterator().next();
    for (final DataSegment segment : segments) {
      if (!segment.getDataSource().equals(firstSegment.getDataSource())
          || !segment.getInterval().equals(firstSegment.getInterval())
          || !segment.getVersion().equals(firstSegment.getVersion())) {
        return false;
      }
    }
    return true;
  }
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
  )
  {
    final LoadQueuePeon toPeon = params.getLoadManagementPeons().get(toServer.getName());

    final ImmutableDruidServer fromServer = segment.getFromServer();
    final DataSegment segmentToMove = segment.getSegment();
    final String segmentName = segmentToMove.getIdentifier();

    if (!toPeon.getSegmentsToLoad().contains(segmentToMove) &&
        (toServer.getSegment(segmentName) == null) &&
        new ServerHolder(toServer, toPeon).getAvailableSize() > segmentToMove.getSize()) {
      log.info("Moving [%s] from [%s] to [%s]", segmentName, fromServer.getName(), toServer.getName());

      LoadPeonCallback callback = null;
      try {
        currentlyMovingSegments.get(toServer.getTier()).put(segmentName, segment);
        callback = new LoadPeonCallback()
        {
          @Override
          public void execute()
          {
            Map<String, BalancerSegmentHolder> movingSegments = currentlyMovingSegments.get(toServer.getTier());
            if (movingSegments != null) {
              movingSegments.remove(segmentName);
            }
          }
        };
        coordinator.moveSegment(
            fromServer,
            toServer,
            segmentToMove.getIdentifier(),
            callback
        );
      }
      catch (Exception e) {
        log.makeAlert(e, String.format("[%s] : Moving exception", segmentName)).emit();
View Full Code Here
493
494
495
496
497
498
499
500
501
502
503
504
505
506
    return dataSources;
  }

  private Pair<DataSegment, Set<String>> getSegment(String segmentId)
  {
    DataSegment theSegment = null;
    Set<String> servers = Sets.newHashSet();
    for (DruidServer druidServer : serverInventoryView.getInventory()) {
      DataSegment currSegment = druidServer.getSegments().get(segmentId);
      if (currSegment != null) {
        theSegment = currSegment;
        servers.add(druidServer.getHost());
      }
    }
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
          }

          // Map merged segment so we can extract dimensions
          final QueryableIndex mappedSegment = IndexIO.loadIndex(fileToUpload);

          final DataSegment segmentToUpload = theSink.getSegment()
                                                     .withDimensions(ImmutableList.copyOf(mappedSegment.getAvailableDimensions()))
                                                     .withBinaryVersion(SegmentUtils.getVersionFromDir(fileToUpload));

          dataSegmentPusher.push(fileToUpload, segmentToUpload);

          log.info(
              "Uploaded segment[%s]",
              segmentToUpload.getIdentifier()
          );

        }
        catch (Exception e) {
          log.warn(e, "Failed to merge and upload");
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
  public BalancerSegmentHolder getRandomBalancerSegmentHolder(final List<ServerHolder> serverHolders)
  {
    final Random rand = new Random();
    ServerHolder fromServerHolder = null;
    DataSegment proposalSegment = null;
    int numSoFar = 0;

    for (ServerHolder server : serverHolders) {
      for (DataSegment segment : server.getServer().getSegments().values()) {
        int randNum = rand.nextInt(numSoFar + 1);
View Full Code Here
TOP

Related Classes of io.druid.timeline.DataSegment

  • io.druid.client.client.BatchServerInventoryViewTest
  • io.druid.client.DataSegmentTest
  • io.druid.client.DirectDruidClientTest
  • io.druid.client.DruidDataSource
  • io.druid.client.DruidServer
  • io.druid.client.indexing.IndexingServiceClient
  • io.druid.client.ServerInventoryView
  • io.druid.indexer.IndexGeneratorJob
  • io.druid.indexer.IndexGeneratorJob$IndexGeneratorReducer
  • io.druid.indexing.common.actions.TaskActionToolbox
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.