Examples of com.asakusafw.runtime.directio.FilePattern

com.asakusafw.runtime.directio.FilePattern

A pattern describes file location.

Pattern Expression

 expression: (segment "/")* segment segment: traverse directory traverse: "**" "/" directory: resource "/" file: resource resource: token (wildcard token)* wildcard? wildcard (token wildcard)* token? token: variable character+ wildcard: "*" variable: "${" character* "}" character: (any character except "/", "\", "$", "*", "?", "#", "{", "}", "[", "]")

For example:

Expression	Description
`a/b/c.csv`	just {@code a/b/c.csv}
`*`	all files in target directory
`**`	all files in target directory (recursive)
`*.csv`	all CSV files in target directory
`*/.csv`	all CSV files in target directory (recursive)

@since 0.2.5 @version 0.7.0

    }


    private boolean checkBasePath(Class<?> theClass, String basePath, String name) {
        boolean valid = true;
        try {
            FilePattern pattern = FilePattern.compile(basePath);
            if (pattern.containsTraverse()) {
                getEnvironment().error(
                        "{0}にワイルドカード (**) を利用できません: {1}",
                        name,
                        theClass.getName());
                valid = false;
            }
            Set<PatternElementKind> kinds = pattern.getPatternElementKinds();
            for (PatternElementKind kind : kinds) {
                if (INVALID_BASE_PATH_KIND.contains(kind)) {
                    getEnvironment().error(
                            "{0}に \"{1}\" を利用できません: {2}",
                            name,

View Full Code Here

        }
        DataFormat<T> format = createFormat(dataType, description.getFormat());
        final DataDefinition<T> definition = SimpleDataDefinition.newInstance(dataType, format);
        final Counter counter = new Counter();
        try {
            FilePattern pattern = toInputPattern(description.getResourcePattern());
            LOG.info("Opening {}/{} for output (id={}, description={})", new Object[] {
                    fullPath,
                    pattern,
                    id,
                    description.getClass().getName(),

View Full Code Here

    }


    private String toOutputName(String inputResourcePattern) throws IOException {
        assert inputResourcePattern != null;
        String patternString = resolve(inputResourcePattern);
        FilePattern pattern = FilePattern.compile(patternString);
        if (pattern.containsVariables()) {
            throw new IOException(MessageFormat.format(
                    "Input resource pattern contains variables (original=[{0}], expanded=[{1}])",
                    inputResourcePattern,
                    patternString));
        }
        StringBuilder buf = new StringBuilder();
        for (Segment segment : pattern.getSegments()) {
            if (buf.length() != 0) {
                buf.append('/');
            }
            if (segment.isTraverse()) {
                buf.append(WILDCARD_REPLACEMENT);

View Full Code Here


        Map<DirectInputGroup, List<InputPath>> results = new HashMap<DirectInputGroup, List<InputPath>>();
        for (StageInput input : inputList) {
            String fullBasePath = variables.parse(extractBasePath(input));
            String basePath = repo.getComponentPath(fullBasePath);
            FilePattern pattern = extractSearchPattern(context, variables, input);
            Class<?> dataClass = extractDataClass(context, input);
            Class<? extends DataFormat<?>> formatClass = extractFormatClass(context, input);
            DirectInputGroup group = new DirectInputGroup(fullBasePath, dataClass, formatClass);
            List<InputPath> paths = results.get(group);
            if (paths == null) {

View Full Code Here

        assert context != null;
        assert input != null;
        String value = extract(input, DirectDataSourceConstants.KEY_RESOURCE_PATH);
        value = variables.parse(value);
        try {
            FilePattern compiled = FilePattern.compile(value);
            if (compiled.containsVariables()) {
                throw new IllegalArgumentException(MessageFormat.format(
                        "Search pattern contains variables: {0}",
                        value));
            }
            return compiled;

View Full Code Here

                OutputTransactionContext context = HadoopDataSourceUtil.createContext(jobContext, id);
                try {
                    DirectDataSource repo = repository.getRelatedDataSource(spec.basePath);
                    String basePath = repository.getComponentPath(spec.basePath);
                    for (String pattern : spec.deletePatterns) {
                        FilePattern resources = FilePattern.compile(pattern);
                        if (LOG.isInfoEnabled()) {
                            LOG.info(MessageFormat.format(
                                    "Deleting output: datasource={0}, basePath={1}, pattern={2}",
                                    id,
                                    basePath,

View Full Code Here

                    "Start finding input (id={0}, path={1}, resourcePattern={2})",
                    profile.getId(),
                    basePath,
                    resourcePattern));
        }
        FilePattern pattern = validate(resourcePattern);
        HadoopDataSourceProfile p = profile;
        FileSystem fs = p.getFileSystem();
        Path root = p.getFileSystemPath();
        Path base = append(root, basePath);
        Path temporary = p.getTemporaryFileSystemPath();

View Full Code Here

                    "Start listing files (id={0}, path={1}, resource={2})",
                    profile.getId(),
                    basePath,
                    resourcePattern));
        }
        FilePattern pattern = validate(resourcePattern);
        HadoopDataSourceProfile p = profile;
        FileSystem fs = p.getFileSystem();
        Path root = p.getFileSystemPath();
        Path base = append(root, basePath);
        Path temporary = p.getTemporaryFileSystemPath();

View Full Code Here

                    profile.getId(),
                    basePath,
                    resourcePattern,
                    recursive));
        }
        FilePattern pattern = validate(resourcePattern);
        HadoopDataSourceProfile p = profile;
        FileSystem fs = p.getFileSystem();
        Path root = p.getFileSystemPath();
        Path base = append(root, basePath);
        List<FileStatus> stats = HadoopDataSourceUtil.search(fs, base, pattern);

View Full Code Here

TOP

Related Classes of com.asakusafw.runtime.directio.FilePattern

com.asakusafw.compiler.directio.DirectFileIoProcessor

com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore

com.asakusafw.runtime.stage.input.BridgeInputFormat

com.asakusafw.runtime.stage.output.BridgeOutputFormat$BridgeOutputCommitter

com.asakusafw.testdriver.directio.DirectIoTestHelper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.