Package org.apache.pig.builtin.mock.Storage

Examples of org.apache.pig.builtin.mock.Storage.Data

91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
        final Statement stmt = conn.createStatement();

        stmt.execute("CREATE TABLE " + tableName +
                 " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR)");

        final Data data = Storage.resetData(pigServer);
        final Collection<Tuple> list = Lists.newArrayList();

        // Create input dataset
        int rows = 100;
        for (int i = 0; i < rows; i++) {
            Tuple t = tupleFactory.newTuple();
            t.append(i);
            t.append("a" + i);
            list.add(t);
        }
        data.set("in", "id:int, name:chararray", list);

        pigServer.setBatchOn();
        pigServer.registerQuery("A = LOAD 'in' USING mock.Storage();");

        pigServer.registerQuery("Store A into 'hbase://" + tableName
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        final Statement stmt = conn.createStatement();

        stmt.execute("CREATE TABLE " + tableName +
                 " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, AGE INTEGER)");

        final Data data = Storage.resetData(pigServer);
        final Collection<Tuple> list = Lists.newArrayList();

        // Create input dataset
        int rows = 100;
        for (int i = 0; i < rows; i++) {
            Tuple t = tupleFactory.newTuple();
            t.append(i);
            t.append("a" + i);
            t.append(i * 2);
            list.add(t);
        }
        data.set("in", "id:int, name:chararray,age:int", list);

        pigServer.setBatchOn();
        pigServer.registerQuery("A = LOAD 'in' USING mock.Storage();");
        pigServer.registerQuery("B = FOREACH A GENERATE id,name;");
        pigServer.registerQuery("Store B into 'hbase://" + tableName + "/ID,NAME"
View Full Code Here
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
            stmt.execute();   
        }
        conn.commit();
       
        //prepare the mock storage with expected output
        final Data data = Storage.resetData(pigServer);
        List<Tuple> expectedList = new ArrayList<Tuple>();
        expectedList.add(Storage.tuple(0,180));
        expectedList.add(Storage.tuple(0,270));
       
         //load data and filter rows whose age is > 25
        pigServer.setBatchOn();
        pigServer.registerQuery(String.format(
                "A = load 'hbase://table/%s' using org.apache.phoenix.pig.PhoenixHBaseLoader('%s');", TABLE_FULL_NAME,
                zkQuorum));
       
        pigServer.registerQuery("B = GROUP A BY AGE;");
        pigServer.registerQuery("C = FOREACH B GENERATE MIN(A.SAL),MAX(A.SAL);");
        pigServer.registerQuery("STORE C INTO 'out' using mock.Storage();");
        pigServer.executeBatch();
       
        List<Tuple> actualList = data.get("out");
        assertEquals(expectedList, actualList);
    }
View Full Code Here
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
        //(Mumbai,20,Applied Math,UAH,Indians,4468)
        //(Praj,54,RMX,Champaign,LNU,0)
        //(Pune,60,Astrophysics,UTA,Warriors,5466)
        //(Rajsathan,20,Biochemistry,Stanford,Royals,1378)

        Data data = resetData(pigServerLocal);
        String output= outbasedir + "testMultipleSchemasWithDefaultValue";
        deleteDirectory(new File(output));
        String expected = basedir + "expected_testMultipleSchemasWithDefaultValue.avro";
        String [] queries = {
          " a = LOAD '" + testMultipleSchemasWithDefaultValue +
              "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ('multiple_schemas');",
          " b = foreach a generate name,age,dept,office,lastname,salary;",
          " c = filter b by age < 40 ;",
          " d = order c by  name;",
          " STORE d INTO '" + output+ "' using mock.Storage();"
           };
        testAvroStorage(queries);
        List<Tuple> out = data.get(output);
        assertEquals(out + " size", 5, out.size());
        assertEquals(
               schema("name: chararray,age: int,dept: chararray,office: chararray,lastname: chararray,salary: int"),
                data.getSchema(output));
        assertEquals(tuple("Asmya", 34, "PQ", "OU", "LNU", 0), out.get(0));
        assertEquals(tuple("Baljit", 23, "RS", "OU", "LNU", 0), out.get(1));
        assertEquals(tuple("Milo", 30, "DH", "OU", "LNU", 0), out.get(2));
        assertEquals(tuple("Mumbai", 20, "Applied Math", "UAH", "Indians", 4468), out.get(3));
        assertEquals(tuple("Rajsathan", 20, "Biochemistry", "Stanford", "Royals", 1378), out.get(4));
View Full Code Here
722
723
724
725
726
727
728
729
730
731
732
733
    // Verify the default values specified in the schema in AvroStorage
    // are actually written to the schema in the output avro file
    public void testDefaultValueSchemaWrite() throws IOException {
        String output = outbasedir + "testDefaultValueSchemaWrite";
        String expected = basedir + "expected_testDefaultSchemaWrite.avro";
        Data data = resetData(pigServerLocal);
              data.set("testDefaultValueSchemaWrite",
                tuple(0,115,115000,115000.1),
                tuple(1,116,116000,116000.1),
                tuple(2,117,117000,117000.1),
                tuple(3,118,118000,118000.1),
                tuple(4,119,119000,119000.1)
View Full Code Here
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
    // {"name":"age","type":["null","int"],"doc":"autogenerated from Pig Field Schema"},
    // {"name":"gpa","type":["null","double"],"doc":"autogenerated from Pig Field Schema"}]}]
    public void testLoadwithNullValues() throws IOException {
    //Input is supposed to have empty tuples
    PigSchema2Avro.setTupleIndex(0);
    Data data = resetData(pigServerLocal);
    String output = outbasedir + "testLoadwithNulls";
    deleteDirectory(new File(output));
    String [] queries = {
       " A = load '" +  testLoadwithNullValues + "' USING " +
          " org.apache.pig.piggybank.storage.avro.AvroStorage(); ",
       " B = order A by name;",
       " store B into '" +  output +"' USING mock.Storage();"
       };
    testAvroStorage(queries);
    List<Tuple> out = data.get(output);
    assertEquals(out + " size", 4, out.size());

    assertEquals(schema("name:chararray,age:int,gpa:double"), data.getSchema(output));

    // sorted data ordered by name
    assertEquals(tuple((String)null),out.get(0));
    assertEquals(tuple((String)null),out.get(1));
    assertEquals(tuple("calvin ellison", 24, 0.71), out.get(2));
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
   * @throws Exception
   */
  @Test
  public void testPositive() throws Exception {
      PigServer pigServer = new PigServer(ExecType.LOCAL);
      Data data = resetData(pigServer);

      data.set("foo",
              tuple(1),
              tuple(2),
              tuple(3)
              );

      pigServer.setBatchOn();
      pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (i:int);");
      pigServer.registerQuery("ASSERT A BY i > 0;");
      pigServer.registerQuery("STORE A INTO 'bar' USING mock.Storage();");
     
      pigServer.executeBatch();

      List<Tuple> out = data.get("bar");
      assertEquals(3, out.size());
      assertEquals(tuple(1), out.get(0));
      assertEquals(tuple(2), out.get(1));
      assertEquals(tuple(3), out.get(2));
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
   * @throws Exception
   */
  @Test
  public void testNegative() throws Exception {
      PigServer pigServer = new PigServer(ExecType.LOCAL);
      Data data = resetData(pigServer);

      data.set("foo",
              tuple(1),
              tuple(2),
              tuple(3)
              );

View Full Code Here
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
    }
   
    @Test
    public void testRetrieveDataFromMap() throws Exception {
        pigServerLocal = new PigServer(ExecType.LOCAL);
        Data data = resetData(pigServerLocal);
        Map<String, String> mapv1 = new HashMap<String, String>();
        mapv1.put("key1", "v11");
        mapv1.put("key2", "v12");
        Map<String, String> mapv2 = new HashMap<String, String>();
        mapv2.put("key1", "v21");
        mapv2.put("key2", "v22");
        data.set("testMap", "maps:map[chararray]", tuple(mapv1), tuple(mapv2));
        String schemaDescription = new String(
                "{" +
                      "\"type\": \"record\"," +
                      "\"name\": \"record\"," +
                      "\"fields\" : [" +
                      "{\"name\" : \"maps\", \"type\" :{\"type\" : \"map\", \"values\" : \"string\"}}" +
                      "]" +
                      "}");
        pigServerLocal.registerQuery("A = LOAD 'testMap' USING mock.Storage();");
        pigServerLocal.registerQuery("STORE A INTO '" + createOutputName() + "' USING AvroStorage('"+ schemaDescription +"');");
        pigServerLocal.registerQuery("B = LOAD '" + createOutputName() + "' USING AvroStorage();");
        pigServerLocal.registerQuery("C = FOREACH B generate maps#'key1';");
        pigServerLocal.registerQuery("STORE C INTO 'out' USING mock.Storage();");
       

        List<Tuple> out = data.get("out");
        assertEquals(tuple("v11"), out.get(0));
        assertEquals(tuple("v21"), out.get(1));
    }
View Full Code Here
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
        //(Mumbai,20,Applied Math,UAH,Indians,4468)
        //(Praj,54,RMX,Champaign,LNU,0)
        //(Pune,60,Astrophysics,UTA,Warriors,5466)
        //(Rajsathan,20,Biochemistry,Stanford,Royals,1378)

        Data data = resetData(pigServerLocal);
        String output= outbasedir + "testMultipleSchemasWithDefaultValue";
        deleteDirectory(new File(output));
        String expected = basedir + "expected_testMultipleSchemasWithDefaultValue.avro";
        String [] queries = {
          " a = LOAD '" + testMultipleSchemasWithDefaultValue +
              "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ('multiple_schemas');",
          " b = foreach a generate name,age,dept,office,lastname,salary;",
          " c = filter b by age < 40 ;",
          " d = order c by  name;",
          " STORE d INTO '" + output+ "' using mock.Storage();"
           };
        testAvroStorage(queries);
        List<Tuple> out = data.get(output);
        assertEquals(out + " size", 5, out.size());
        assertEquals(
               schema("name: chararray,age: int,dept: chararray,office: chararray,lastname: chararray,salary: int"),
                data.getSchema(output));
        assertEquals(tuple("Asmya", 34, "PQ", "OU", "LNU", 0), out.get(0));
        assertEquals(tuple("Baljit", 23, "RS", "OU", "LNU", 0), out.get(1));
        assertEquals(tuple("Milo", 30, "DH", "OU", "LNU", 0), out.get(2));
        assertEquals(tuple("Mumbai", 20, "Applied Math", "UAH", "Indians", 4468), out.get(3));
        assertEquals(tuple("Rajsathan", 20, "Biochemistry", "Stanford", "Royals", 1378), out.get(4));
View Full Code Here
TOP

Related Classes of org.apache.pig.builtin.mock.Storage.Data

  • org.apache.phoenix.pig.PhoenixHBaseLoaderIT
  • org.apache.phoenix.pig.PhoenixHBaseStorerIT
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.TestJobControlSleep
  • org.apache.pig.builtin.mock.TestMockStorage
  • org.apache.pig.builtin.TestAvroStorage
  • org.apache.pig.builtin.TestBigTypeSort
  • org.apache.pig.builtin.TestCurrentTime
  • org.apache.pig.builtin.TestInvokerGenerator
  • org.apache.pig.builtin.TestPluckTuple
  • org.apache.pig.data.TestSchemaTuple
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.