Package com.asakusafw.runtime.directio

Examples of com.asakusafw.runtime.directio.FilePattern$Variable


    }

    private boolean checkBasePath(Class<?> theClass, String basePath, String name) {
        boolean valid = true;
        try {
            FilePattern pattern = FilePattern.compile(basePath);
            if (pattern.containsTraverse()) {
                getEnvironment().error(
                        "{0}にワイルドカード (**) を利用できません: {1}",
                        name,
                        theClass.getName());
                valid = false;
            }
            Set<PatternElementKind> kinds = pattern.getPatternElementKinds();
            for (PatternElementKind kind : kinds) {
                if (INVALID_BASE_PATH_KIND.contains(kind)) {
                    getEnvironment().error(
                            "{0}に \"{1}\" を利用できません: {2}",
                            name,
View Full Code Here


        }
        DataFormat<T> format = createFormat(dataType, description.getFormat());
        final DataDefinition<T> definition = SimpleDataDefinition.newInstance(dataType, format);
        final Counter counter = new Counter();
        try {
            FilePattern pattern = toInputPattern(description.getResourcePattern());
            LOG.info("Opening {}/{} for output (id={}, description={})", new Object[] {
                    fullPath,
                    pattern,
                    id,
                    description.getClass().getName(),
View Full Code Here

    }

    private String toOutputName(String inputResourcePattern) throws IOException {
        assert inputResourcePattern != null;
        String patternString = resolve(inputResourcePattern);
        FilePattern pattern = FilePattern.compile(patternString);
        if (pattern.containsVariables()) {
            throw new IOException(MessageFormat.format(
                    "Input resource pattern contains variables (original=[{0}], expanded=[{1}])",
                    inputResourcePattern,
                    patternString));
        }
        StringBuilder buf = new StringBuilder();
        for (Segment segment : pattern.getSegments()) {
            if (buf.length() != 0) {
                buf.append('/');
            }
            if (segment.isTraverse()) {
                buf.append(WILDCARD_REPLACEMENT);
View Full Code Here

        Map<DirectInputGroup, List<InputPath>> results = new HashMap<DirectInputGroup, List<InputPath>>();
        for (StageInput input : inputList) {
            String fullBasePath = variables.parse(extractBasePath(input));
            String basePath = repo.getComponentPath(fullBasePath);
            FilePattern pattern = extractSearchPattern(context, variables, input);
            Class<?> dataClass = extractDataClass(context, input);
            Class<? extends DataFormat<?>> formatClass = extractFormatClass(context, input);
            DirectInputGroup group = new DirectInputGroup(fullBasePath, dataClass, formatClass);
            List<InputPath> paths = results.get(group);
            if (paths == null) {
View Full Code Here

        assert context != null;
        assert input != null;
        String value = extract(input, DirectDataSourceConstants.KEY_RESOURCE_PATH);
        value = variables.parse(value);
        try {
            FilePattern compiled = FilePattern.compile(value);
            if (compiled.containsVariables()) {
                throw new IllegalArgumentException(MessageFormat.format(
                        "Search pattern contains variables: {0}",
                        value));
            }
            return compiled;
View Full Code Here

                OutputTransactionContext context = HadoopDataSourceUtil.createContext(jobContext, id);
                try {
                    DirectDataSource repo = repository.getRelatedDataSource(spec.basePath);
                    String basePath = repository.getComponentPath(spec.basePath);
                    for (String pattern : spec.deletePatterns) {
                        FilePattern resources = FilePattern.compile(pattern);
                        if (LOG.isInfoEnabled()) {
                            LOG.info(MessageFormat.format(
                                    "Deleting output: datasource={0}, basePath={1}, pattern={2}",
                                    id,
                                    basePath,
View Full Code Here

                    "Start finding input (id={0}, path={1}, resourcePattern={2})",
                    profile.getId(),
                    basePath,
                    resourcePattern));
        }
        FilePattern pattern = validate(resourcePattern);
        HadoopDataSourceProfile p = profile;
        FileSystem fs = p.getFileSystem();
        Path root = p.getFileSystemPath();
        Path base = append(root, basePath);
        Path temporary = p.getTemporaryFileSystemPath();
View Full Code Here

                    "Start listing files (id={0}, path={1}, resource={2})",
                    profile.getId(),
                    basePath,
                    resourcePattern));
        }
        FilePattern pattern = validate(resourcePattern);
        HadoopDataSourceProfile p = profile;
        FileSystem fs = p.getFileSystem();
        Path root = p.getFileSystemPath();
        Path base = append(root, basePath);
        Path temporary = p.getTemporaryFileSystemPath();
View Full Code Here

                    profile.getId(),
                    basePath,
                    resourcePattern,
                    recursive));
        }
        FilePattern pattern = validate(resourcePattern);
        HadoopDataSourceProfile p = profile;
        FileSystem fs = p.getFileSystem();
        Path root = p.getFileSystemPath();
        Path base = append(root, basePath);
        List<FileStatus> stats = HadoopDataSourceUtil.search(fs, base, pattern);
View Full Code Here

        List<GradsVariable>  vars  = gradsDDF.getVariables();
        List<GradsAttribute> attrs = gradsDDF.getAttributes();
        //TODO: ensembles
        List<GradsDimension>       dims = gradsDDF.getDimensions();
        Variable                   v;
        int                        numZ  = 0;
        HashMap<String, Dimension> zDims = new HashMap<String, Dimension>();
        for (GradsDimension dim : dims) {
            String    name  = getVarName(dim);
            int       size  = dim.getSize();
            Dimension ncDim = new Dimension(name, size, true);
            ncFile.addDimension(null, ncDim);
            if (name.equals(ENS_VAR)) {
                v = new Variable(ncFile, null, null, name, DataType.STRING,
                                 name);
                v.addAttribute(new Attribute("standard_name", "ensemble"));
                v.addAttribute(new Attribute(_Coordinate.AxisType,
                                             AxisType.Ensemble.toString()));
                List<String> names =
                    gradsDDF.getEnsembleDimension().getEnsembleNames();
                String[] nameArray = new String[names.size()];
                for (int i = 0; i < nameArray.length; i++) {
                    nameArray[i] = names.get(i);
                }
                Array dataArray = Array.factory(DataType.STRING,
                                      new int[] { nameArray.length },
                                      nameArray);
                v.setCachedData(dataArray, false);
            } else {
                double[] vals = dim.getValues();
                v = new Variable(ncFile, null, null, name, DataType.DOUBLE,
                                 name);
                v.addAttribute(new Attribute("units", dim.getUnit()));
                if (name.equals(Y_VAR)) {
                    v.addAttribute(new Attribute("long_name", "latitude"));
                    v.addAttribute(new Attribute("standard_name",
                            "latitude"));
                    v.addAttribute(new Attribute("axis", "Y"));
                    sizeY = dim.getSize();
                    v.addAttribute(new Attribute(_Coordinate.AxisType,
                            AxisType.Lat.toString()));
                } else if (name.equals(X_VAR)) {
                    v.addAttribute(new Attribute("long_name", "longitude"));
                    v.addAttribute(new Attribute("standard_name",
                            "longitude"));
                    v.addAttribute(new Attribute("axis", "X"));
                    v.addAttribute(new Attribute(_Coordinate.AxisType,
                            AxisType.Lon.toString()));
                    sizeX = dim.getSize();
                } else if (name.equals(Z_VAR)) {
                    numZ = size;
                    zDims.put(name, ncDim);
                    v.addAttribute(new Attribute("long_name", "level"));
                    addZAttributes(dim, v);
                } else if (name.equals(TIME_VAR)) {
                    v.addAttribute(new Attribute("long_name", "time"));
                    v.addAttribute(new Attribute(_Coordinate.AxisType,
                            AxisType.Time.toString()));
                }
                ArrayDouble.D1 varArray = new ArrayDouble.D1(size);
                for (int i = 0; i < vals.length; i++) {
                    varArray.set(i, vals[i]);
                }
                v.setCachedData(varArray, false);
            }
            ncFile.addVariable(null, v);
        }
        if (numZ > 0) {
            GradsDimension zDim = gradsDDF.getZDimension();
            double[]       vals = zDim.getValues();
            for (GradsVariable var : vars) {
                int nl = var.getNumLevels();
                if ((nl > 0) && (nl != numZ)) {
                    String name = Z_VAR + nl;
                    if (zDims.get(name) == null) {
                        Dimension ncDim = new Dimension(name, nl, true);
                        ncFile.addDimension(null, ncDim);
                        Variable vz = new Variable(ncFile, null, null, name,
                                          DataType.DOUBLE, name);
                        vz.addAttribute(new Attribute("long_name", name));
                        vz.addAttribute(new Attribute("units",
                                zDim.getUnit()));
                        addZAttributes(zDim, vz);
                        ArrayDouble.D1 varArray = new ArrayDouble.D1(nl);
                        for (int i = 0; i < nl; i++) {
                            varArray.set(i, vals[i]);
                        }
                        vz.setCachedData(varArray, false);
                        ncFile.addVariable(null, vz);
                        zDims.put(name, ncDim);
                    }
                }
            }
        }
        zDims = null;
        for (GradsVariable var : vars) {
            String coords = "latitude longitude";
            int    nl     = var.getNumLevels();
            if (nl > 0) {
                if (nl == numZ) {
                    coords = "level " + coords;
                } else {
                    coords = Z_VAR + nl + " " + coords;
                }
            }
            coords = "time " + coords;
            if (gradsDDF.getEnsembleDimension() != null) {
                coords = "ensemble " + coords;
            }
            v = new Variable(ncFile, null, null, var.getName(),
                             DataType.FLOAT, coords);
            v.addAttribute(new Attribute("long_name", var.getDescription()));
            if (var.getUnitName() != null) {
                v.addAttribute(new Attribute("units", var.getUnitName()));
            }
View Full Code Here

TOP

Related Classes of com.asakusafw.runtime.directio.FilePattern$Variable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.