Package org.apache.oozie.util

Examples of org.apache.oozie.util.HCatURI

89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
        setupServices("testevictionontimetolive");
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        int numItems = 50;
        long startTime = System.currentTimeMillis();
        for (int i = 0; i < numItems; i++) {
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
            pdms.addMissingDependency(dep, "" + i);
        }
        verifyWaitingAction(pdms, numItems);
        LOG.info("Time taken to insert and retrive " + numItems + " items is "
                + (System.currentTimeMillis() - startTime));
        // timeToLiveSeconds is 1
        Thread.sleep(1100);
        for (int i = 0; i < numItems; i++) {
            assertNull(pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)));
        }
    }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    public void testMaxElementsInMemory() throws Exception {
        setupServices("testmaxelementsinmemory"); // maxElementsInMemory="500" overflowToDisk="false"
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        int numItems = 1000;
        for (int i = 0; i < numItems; i++) {
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
            pdms.addMissingDependency(dep, "" + i);
        }
        // First 500 should have been evicted. But it is LRU and the last 350 removed is between 250 and 750.
        for (int i = 0; i < 150; i++) {
            assertNull(pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)));
        }
        int evicted = 0;
        for (int i = 150; i < 750; i++) {
            if (pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)) == null) {
                evicted++;
            }
        }
        assertEquals(350, evicted);
        for (int i = 750; i < 1000; i++) {
            String actionID = "" + i;
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + actionID);
            Collection<String> waitingActions = pdms.getWaitingActions(dep);
            assertNotNull(dep.toURIString() + " is missing in cache", waitingActions);
            assertTrue(dep.toURIString() + " is missing in cache", waitingActions.contains(actionID));
        }
    }
View Full Code Here
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
        populateTable(db, table);
        String actionId = addInitRecords(newHCatDependency);
        CoordinatorAction ca = checkCoordActionDependencies(actionId, newHCatDependency);
        assertEquals(CoordinatorAction.Status.WAITING, ca.getStatus());
        // Register the missing dependencies to PDMS assuming CoordPushDependencyCheckCommand did this.
        pdms.addMissingDependency(new HCatURI(newHCatDependency1), actionId);
        pdms.addMissingDependency(new HCatURI(newHCatDependency2), actionId);

        sleep(2000);
        Runnable recoveryRunnable = new RecoveryRunnable(0, 1, 1);
        recoveryRunnable.run();
        sleep(2000);

        // Recovery service should have discovered newHCatDependency2 and JMS Connection should exist
        // and newHCatDependency1 should be in PDMS waiting list
        assertTrue(jmsService.isListeningToTopic(hcatService.getJMSConnectionInfo(new URI(newHCatDependency2)), "hcat."
                + db + "." + table));
        checkCoordActionDependencies(actionId, newHCatDependency1);

        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency2)));
        Collection<String> waitingActions = pdms.getWaitingActions(new HCatURI(newHCatDependency1));
        assertEquals(1, waitingActions.size());
        assertTrue(waitingActions.contains(actionId));
    }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
147
148
     *
     * @param dataInName
     * @return DB name
     */
    public static String ph3_coord_databaseIn(String dataName) {
        HCatURI hcatURI = getURIFromResolved(dataName, EventType.input);
        if (hcatURI != null) {
            return hcatURI.getDb();
        }
        else {
            return "";
        }
    }
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
166
167
     *
     * @param dataOutName
     * @return DB name
     */
    public static String ph3_coord_databaseOut(String dataName) {
        HCatURI hcatURI = getURIFromResolved(dataName, EventType.output);
        if (hcatURI != null) {
            return hcatURI.getDb();
        }
        else {
            return "";
        }
    }
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
185
186
     *
     * @param dataInName
     * @return Table name
     */
    public static String ph3_coord_tableIn(String dataName) {
        HCatURI hcatURI = getURIFromResolved(dataName, EventType.input);
        if (hcatURI != null) {
            return hcatURI.getTable();
        }
        else {
            return "";
        }
    }
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
205
     *
     * @param dataOutName
     * @return Table name
     */
    public static String ph3_coord_tableOut(String dataName) {
        HCatURI hcatURI = getURIFromResolved(dataName, EventType.output);
        if (hcatURI != null) {
            return hcatURI.getTable();
        }
        else {
            return "";
        }
    }
View Full Code Here
238
239
240
241
242
243
244
245
246
247
248
249
        Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
        if (unresolved != null && unresolved.booleanValue() == true) {
            return "${coord:dataOutPartitionValue('" + dataOutName + "', '" + partitionName + "')}";
        }
        try {
            HCatURI hcatUri = new HCatURI(uri);
            return hcatUri.getPartitionValue(partitionName);
        }
        catch(URISyntaxException urie) {
            XLog.getLog(HCatELFunctions.class).warn("Exception with uriTemplate [{0}]. Reason [{1}]: ", uri, urie);
            throw new RuntimeException("HCat URI can't be parsed " + urie);
        }
View Full Code Here
263
264
265
266
267
268
269
270
271
272
273
        Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
        if (unresolved != null && unresolved.booleanValue() == true) {
            return "${coord:dataOutPartitions('" + dataOutName + "')}";
        }
        try {
            return new HCatURI(uri).toPartitionString();
        }
        catch (URISyntaxException e) {
            throw new RuntimeException("Parsing exception for HCatURI " + uri + ". details: " + e);
        }
    }
View Full Code Here
295
296
297
298
299
300
301
302
303
304
305
                if (uriList.length > 1) {
                    throw new RuntimeException("Multiple partitions not supported for hive-export type. Dataset name: "
                        + dataInName + " URI: " + uri);
                }
                try {
                    partitionValue = new HCatURI(uri).toPartitionValueString(type);
                }
                catch (URISyntaxException e) {
                    throw new RuntimeException("Parsing exception for HCatURI " + uri, e);
                }
            } else {
View Full Code Here
TOP

Related Classes of org.apache.oozie.util.HCatURI

  • org.apache.oozie.action.hadoop.HCatLauncherURIHandler
  • org.apache.oozie.command.coord.TestCoordActionUpdatePushMissingDependency
  • org.apache.oozie.command.coord.TestCoordKillXCommand
  • org.apache.oozie.command.coord.TestCoordPushDependencyCheckXCommand
  • org.apache.oozie.coord.HCatELFunctions
  • org.apache.oozie.dependency.hcat.EhcacheHCatDependencyCache
  • org.apache.oozie.dependency.hcat.SimpleHCatDependencyCache
  • org.apache.oozie.dependency.HCatURIHandler
  • org.apache.oozie.jms.TestHCatMessageHandler
  • org.apache.oozie.service.TestHAPartitionDependencyManagerService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.