Package com.mapr.storm

Examples of com.mapr.storm.DirectoryScanner

109
110
111
112
113
114
115
116
        this.factory = factory;
        this.statusFile = statusFile;
        if (statusFile.exists() && statusFile.length() > 0) {
            scanner = SpoutState.restoreState(pendingReplays, statusFile);
        } else {
            scanner = new DirectoryScanner(inputDirectory, inputFileNamePattern);
        }
    }
View Full Code Here
100
101
102
103
104
105
106
    public DirectoryObserver(StreamParserFactory factory, File statusFile,
                             File inputDirectory, final Pattern inputFileNamePattern) {
        this.factory = factory;
        this.statusFile = statusFile;
        scanner = new DirectoryScanner(inputDirectory, inputFileNamePattern);
    }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
    @Test
    public void testSimpleFileRoll() throws IOException, InterruptedException {
        tempDir = Files.createTempDir();

        DirectoryScanner scanner = new DirectoryScanner(tempDir, Pattern.compile("x-.*"));

        // add some files
        Files.append("4\n5\n6\n", new File(tempDir, "x-2"), Charsets.UTF_8);
        Files.append("1\n2\n3\n", new File(tempDir, "x-1"), Charsets.UTF_8);

        // verify we read both files in order
        List<String> lines1 = CharStreams.readLines(new InputStreamReader(scanner.nextInput()));
        assertEquals(3, lines1.size());
        assertEquals("[1, 2, 3]", lines1.toString());

        List<String> lines2 = CharStreams.readLines(new InputStreamReader(scanner.nextInput()));
        assertEquals(3, lines2.size());
        assertEquals("[4, 5, 6]", lines2.toString());

        // verify that we get empty records for a bit
        FileInputStream in = scanner.nextInput();
        Assert.assertNull(in);

        in = scanner.nextInput();
        Assert.assertNull(in);

        // add another file
        Files.append("7\n8\n", new File(tempDir, "x-3"), Charsets.UTF_8);

        List<String> lines3 = CharStreams.readLines(new InputStreamReader(scanner.nextInput()));
        assertEquals(2, lines3.size());
        assertEquals("[7, 8]", lines3.toString());

        // delete an old file without a problem
        new File(tempDir, "x-1").delete();

        in = scanner.nextInput();
        Assert.assertNull(in);

        // add a file that doesn't match the pattern without impact
        Files.append("9\n10\n", new File(tempDir, "y-1"), Charsets.UTF_8);

        in = scanner.nextInput();
        Assert.assertNull(in);

        // out of order collation
        Files.append("9\n10\n", new File(tempDir, "x-10"), Charsets.UTF_8);

        List<String> lines4 = CharStreams.readLines(new InputStreamReader(scanner.nextInput()));
        assertEquals(2, lines4.size());
        assertEquals("[9, 10]", lines4.toString());
    }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
        inputDir = Files.createTempDir();
        File statusFile = File.createTempFile("state", ".json");
        statusFile.deleteOnExit();

        Map<Long, PendingMessage> emptyAcks = Maps.newHashMap();
        final DirectoryScanner scanner = new DirectoryScanner(inputDir, Pattern.compile(".*\\.log"));

        final File logFile = new File(inputDir, "test");
        Files.write("1\n2\n3\n", logFile, Charsets.UTF_8);
        scanner.forceInput(logFile, 2);

        SpoutState.recordCurrentState(emptyAcks, scanner, new FakeParser(), statusFile);
        SpoutState rs = SpoutState.fromString(Files.toString(statusFile, Charsets.UTF_8));
        assertEquals(0, rs.getOldFiles().size());
        assertEquals(1, rs.getOffsets().size());
        Map.Entry<File, Long> entry = rs.getOffsets().entrySet().iterator().next();
        assertEquals(10247, (long) entry.getValue());
        assertEquals("test", entry.getKey().getName());
        assertEquals(".*\\.log", rs.getFilePattern().toString());

        Queue<PendingMessage> q = Lists.newLinkedList();
        DirectoryScanner s2 = SpoutState.restoreState(q, statusFile);
        assertEquals(1, q.size());
        assertEquals(0, s2.getOldFiles().size());
        assertEquals(scanner.getFileNamePattern().toString(), s2.getFileNamePattern().toString());
        assertEquals(scanner.getInputDirectory().toString(), s2.getInputDirectory().toString());
    }
View Full Code Here
TOP

Related Classes of com.mapr.storm.DirectoryScanner

  • com.mapr.DirectoryObserver
  • com.mapr.storm.test.DirectoryScannerTest
  • com.mapr.storm.test.SpoutStateTest
  • com.mapr.TailSpout
  • java.io.File
  • java.io.FilenameFilter
  • java.io.FileInputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.