Package com.splout.db.common

Examples of com.splout.db.common.SploutClient

40
41
42
43
44
45
46
47
      return new SploutState(failFast, qNodes);
    }
  }

  public SploutState(boolean failFast, String... qNodes) {
    this.sploutClient = new SploutClient(qNodes);
    this.failFast = failFast;
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
      t.printStackTrace();
      jComm.usage();
      return -1;
    }

    SploutClient client = new SploutClient(qnode);
    List<String> dnodes = client.dNodeList();
    final int nDNodes = dnodes.size();

    // Kill DNodes every X seconds
    try {
      while(true) {
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
        "projectcode:string, pagename:string, visits:int, bytes:long", "-pby", "projectcode,pagename",
        "-sep", "\" \"", "-p", "2", "-e", engine }) < 0) {
      throw new RuntimeException("Generator failed!");
    }

    SploutClient client = new SploutClient(qnode);
    QNodeStatus status = client.overview();
    long previousVersion = -1;
    if(status.getTablespaceMap().get("pagecountsintegration") != null) {
      previousVersion = status.getTablespaceMap().get("pagecountsintegration").getVersion();
    }

    DeployerCMD deployer = new DeployerCMD();
    deployer.setConf(getConf());
    if(deployer.run(new String[] { "-r", "2", "-q", qnode, "-root", tmpHdfsPath + "/output", "-ts",
        "pagecountsintegration" }) < 0) {
      throw new RuntimeException("Deployer failed!");
    }

    long waitedSoFar = 0;

    status = client.overview();
    while(status.getTablespaceMap().get("pagecountsintegration") == null
        || previousVersion == status.getTablespaceMap().get("pagecountsintegration").getVersion()) {
      Thread.sleep(2000);
      waitedSoFar += 2000;
      status = client.overview();
      if(waitedSoFar > 90000) {
        throw new RuntimeException(
            "Deploy must have failed in Splout's server. Waiting too much for it to complete.");
      }
    }

    previousVersion = status.getTablespaceMap().get("pagecountsintegration").getVersion();

    QueryStatus qStatus = client.query("pagecountsintegration", "*", "SELECT * FROM pagecounts;", null);
    System.out.println(qStatus.getResult());

    if(qStatus.getResult() == null) {
      throw new RuntimeException("Something failed as query() is returning null!");
    }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
    }

    final SploutClient[] clients = new SploutClient[N_QNODES];

    for(int i = 0; i < N_QNODES; i++) {
      clients[i] = new SploutClient(getqNodes().get(i).getAddress());
    }
    final SploutClient client1 = clients[0];

    // Check that all QNodes have the full list of DNodes
    new TestUtils.NotWaitingForeverCondition() {
      @Override
      public boolean endCondition() {
        try {
          for(int i = 0; i < N_QNODES; i++) {
            List<String> dNodeList = clients[i].dNodeList();
            if(dNodeList.size() != 3) {
              return false;
            }
            QNodeHandler handler = (QNodeHandler)getqNodes().get(i).getHandler();
            for(String dnode: dNodeList) {
              if(handler.getContext().getThriftClientCache().get(dnode) == null) {
                return false;
              }
            }
          }
          return true;
        } catch(IOException e) {
          // test failed
          e.printStackTrace();
          return true;
        }
      }
    }.waitAtMost(5000);
   
    // Deploy
    client1.deploy("p1", testTablespace.getPartitionMap(), testTablespace.getReplicationMap(),
        deployData.getAbsoluteFile().toURI());

    // Check that all QNodes have the deployment data
    new TestUtils.NotWaitingForeverCondition() {
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
      }
      manager.close();
    }

    SploutConfiguration config = SploutConfiguration.getTestConfig();
    SploutClient client = getRandomQNodeClient(random, config);
    client.deploy("p1", testTablespace.getPartitionMap(), testTablespace.getReplicationMap(),
        deployData.getAbsoluteFile().toURI());

    Thread.sleep(2000); // TODO How to improve this.

    // Perform N queries, one to each DNode and validate the resultant data
    for(int i = 0; i < N_DNODES; i++) {
      client = getRandomQNodeClient(random, config);
      QueryStatus qStatus = client.query("p1", ((i * 10) + 1) + "", "SELECT * FROM foo;", null);
      assertEquals((Integer) i, qStatus.getShard());
      assertEquals(10, qStatus.getResult().size());
      for(Object obj : qStatus.getResult()) {
        Map<String, Object> map = (Map<String, Object>) obj;
        assertEquals(randomStr, map.get("strCol"));
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
    // We now query for the alive DNodes and build deployRequests accordingly
    DeployRequest[] deployRequests = new DeployRequest[deployments.size()];

    log.info("Querying Splout QNode for list of DNodes...");
    SploutClient client = new SploutClient(qnode);
    List<String> dnodes = client.dNodeList();
    if(dnodes == null || dnodes.size() == 0) {
      throw new IOException("No available DNodes in Splout cluster.");
    }
   
    int tIndex = 0;
    for (TablespaceDepSpec tablespace : deployments) {
      Path tablespaceOut = new Path(tablespace.getSourcePath());

      // Define a DeployRequest for this Tablespace
      deployRequests[tIndex] = new DeployRequest();

      // Splout only accepts absolute URIs
      FileSystem sourceFs = tablespaceOut.getFileSystem(conf);
      if (!sourceFs.exists(tablespaceOut)) {
        throw new IllegalArgumentException("Folder doesn't exist: " + tablespaceOut);
      }
      Path absoluteOutPath = tablespaceOut.makeQualified(sourceFs);

      Path partitionMapPath = new Path(tablespaceOut, TablespaceGenerator.OUT_PARTITION_MAP);
      if (!sourceFs.exists(partitionMapPath)) {
        throw new IllegalArgumentException("Invalid tablespace folder: " + tablespaceOut + " doesn't contain a partition-map file.");
      }

      // Load the partition map
      PartitionMap partitionMap = JSONSerDe.deSer(
          HadoopUtils.fileToString(sourceFs, partitionMapPath), PartitionMap.class);

      // Load the init statements, if they exist
      ArrayList<String> initStatements = new ArrayList<String>();
      Path initStatementsPath = new Path(tablespaceOut, TablespaceGenerator.OUT_INIT_STATEMENTS);
      if (sourceFs.exists(initStatementsPath)) {
        initStatements.addAll(JSONSerDe.deSer(HadoopUtils.fileToString(sourceFs, initStatementsPath), ArrayList.class));
      }
      // Add the other initStatements coming in the deploy request
      if (tablespace.getInitStatements() != null) {
        initStatements.addAll(tablespace.getInitStatements());
      }
     
      String engine = DefaultEngine.class.getName();
      // New : load the engine id used in the generation tool, if exists ( to maintain backwards compatibility )
      Path engineId = new Path(tablespaceOut, TablespaceGenerator.OUT_ENGINE);
      if(sourceFs.exists(engineId)) {
        engine = HadoopUtils.fileToString(sourceFs, engineId);
        log.info("Using generated engine id: " + engine);
      }
     
      // Finally set
      deployRequests[tIndex].setInitStatements(initStatements);
      deployRequests[tIndex].setEngine(engine);
     
      deployRequests[tIndex].setTablespace(tablespace.getTablespace());
      deployRequests[tIndex].setData_uri(new Path(absoluteOutPath, "store").toUri().toString());
      deployRequests[tIndex].setPartitionMap(partitionMap.getPartitionEntries());

      // If rep > dnodes, impossible to reach this level of replication
      int repFactor = tablespace.getReplication();
      if (dnodes.size() < repFactor) {
        log.warn("WARNING: Replication factor " + repFactor + " for tablespace " + tablespace.getTablespace() + " is bigger than the number of serving DNodes. Adjusting replication factor to " + dnodes.size());
        repFactor = dnodes.size();
      }

      deployRequests[tIndex].setReplicationMap(ReplicationMap.roundRobinMap(partitionMap.getPartitionEntries().size(), repFactor,
          dnodes.toArray(new String[0])).getReplicationEntries());

      tIndex++;
    }

    // Finally we send the deploy request
    DeployInfo dInfo = client.deploy(deployRequests);

    log.info("Deploy request of [" + deployments.size() + "] tablespaces performed. Deploy on [" + qnode + "] with version [" + dInfo.getVersion() + "] in progress.");
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
    context.put("qnodes", qNodes);
    Map<Integer, Long> rowIdsPerPartition = new HashMap<Integer, Long>();
    context.put("rowIdsPerPartition", rowIdsPerPartition);

    // First get Tablespace metadata - nPartitions
    SploutClient client = new SploutClient(((String) context.get("qnodes")).split(","));
    Tablespace tablespace = client.tablespace(TABLESPACE);
    int nPartitions = tablespace.getPartitionMap().getPartitionEntries().size();

    // Then gather number of registers for each partition
    for(int partition = 0; partition < nPartitions; partition++) {
      String query = "SELECT MAX(rowid) FROM pagecounts;";
      Map<String, Object> obj = (Map<String, Object>) client
          .query(TABLESPACE, null, query, partition + "").getResult().get(0);
      rowIdsPerPartition.put(partition, Long.parseLong(obj.get("MAX(rowid)").toString()));
    }

    SploutBenchmark benchmark = new SploutBenchmark();
View Full Code Here
93
94
95
96
97
98
99
100
    int querySequence = 0;
    int partition;

    @Override
    public void init(Map<String, Object> context) throws Exception {
      client = new SploutClient(((String) context.get("qnodes")).split(","));
      rowIdsPerPartition = (Map<Integer, Long>) context.get("rowIdsPerPartition");
    }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
    @SuppressWarnings("unchecked")
    @Override
    public void init(Map<String, Object> context) throws Exception {
      this.context = context;
      client = new SploutClient(20*1000, ((String)context.get("qnodes")).split(","));
      String query = "SELECT stn, wban FROM stations";
      System.out.println("Retrieving stations list");
      long statTime = System.currentTimeMillis();
      QueryStatus st = client.query(tablespace, "any", query, null);
      if(st.getResult() == null) {
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    int minKey, maxKey;
    String paddingExp;
   
    @Override
    public void init(Map<String, Object> context) throws Exception {
      client = new SploutClient(((String)context.get("qnodes")).split(","));
      minKey = (Integer) context.get("minKey");
      maxKey = (Integer) context.get("maxKey");
      paddingExp = (String) context.get("paddingExp");
      tablespace = (String) context.get("tablespace");
    }
View Full Code Here
TOP

Related Classes of com.splout.db.common.SploutClient

  • com.datasalt.trident.SploutState
  • com.google.api.client.http.GenericUrl
  • com.google.api.client.http.HttpContent
  • com.google.api.client.http.HttpRequest
  • com.google.api.client.http.HttpRequestInitializer
  • com.google.api.client.http.HttpResponse
  • com.google.api.client.http.HttpTransport
  • com.google.api.client.http.javanet.NetHttpTransport
  • com.splout.db.benchmark.BenchmarkTool$BenchmarkToolStressThreadImpl
  • com.splout.db.examples.MeteoBenchmark$MeteoStressThreadImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.