Examples of com.asakusafw.runtime.directio.DirectInputFragment

com.asakusafw.runtime.directio.DirectInputFragment
The fragment of each input dataset. @since 0.2.5

                    false);
            // TODO configurable split
            for (StripeInformation stripe : orc.getStripes()) {
                long begin = stripe.getOffset();
                long end = begin + stripe.getLength();
                DirectInputFragment fragment = blockMap.get(begin, end);
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format(
                            "Detect ORCFile stripe: path={0}, rows={1}, range={2}+{3}, allocation={4}",
                            fragment.getPath(),
                            stripe.getNumberOfRows(),
                            fragment.getOffset(),
                            fragment.getSize(),
                            fragment.getOwnerNodeNames()));
                }
                results.add(fragment);
            }
        }
        return results;

View Full Code Here

                    begin = Math.min(begin, offset);
                    end = Math.max(end, offset + size);
                }
                assert begin >= 0;
                assert end >= 0;
                DirectInputFragment fragment = blockMap.get(begin, end);
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format(
                            "Detect Parquet file block: path={0}, rows={1}, range={2}+{3}, allocation={4}",
                            status.getPath(),
                            block.getRowCount(),
                            begin,
                            end - begin,
                            fragment.getOwnerNodeNames()));
                }
                results.add(fragment);
            }
        }
        return results;

View Full Code Here

                Arrays.asList(stat), fs,
                -1L, -1L,
                false, false));


        assertThat(fragments, hasSize(1));
        DirectInputFragment first = fragments.get(0);


        ModelInput<MockSimple> input = format.createInput(
                MockSimple.class,
                fs, new Path(first.getPath()),
                first.getOffset(), first.getSize(),
                new Counter());
        try {
            MockSimple buf = new MockSimple();
            assertThat(input.readTo(buf), is(true));
            assertThat(buf.number, is(new IntOption(100)));

View Full Code Here

                Arrays.asList(stat), fs,
                -1L, -1L,
                false, false));


        assertThat(fragments, hasSize(1));
        DirectInputFragment first = fragments.get(0);


        ModelInput<MockSimple> input = format.createInput(
                MockSimple.class,
                fs, new Path(first.getPath()),
                first.getOffset(), first.getSize(),
                new Counter());
        try {
            MockSimple buf = new MockSimple();
            assertThat(input.readTo(buf), is(true));
            assertThat(buf.number, is(new IntOption(100)));

View Full Code Here

                    while (true) {
                        if (current == null) {
                            if (iterator.hasNext() == false) {
                                return false;
                            }
                            DirectInputFragment fragment = iterator.next();
                            try {
                                current = dataSource.openInput(definition, fragment, counter);
                            } catch (InterruptedException e) {
                                throw (IOException) new InterruptedIOException("interrupted").initCause(e);
                            }

View Full Code Here

     * @param end the end offset (exclusive)
     * @return the computed fragment
     */
    public DirectInputFragment get(long start, long end) {
        List<String> hosts = computeHosts(start, end);
        return new DirectInputFragment(path, start, end - start, hosts);
    }

View Full Code Here

            DirectInputGroup groupCopy = group;
            WritableUtils.writeString(out, groupCopy.containerPath);
            WritableUtils.writeString(out, groupCopy.dataType.getName());
            WritableUtils.writeString(out, groupCopy.formatClass.getName());


            DirectInputFragment fragmentCopy = fragment;
            WritableUtils.writeString(out, fragmentCopy.getPath());
            WritableUtils.writeVLong(out, fragmentCopy.getOffset());
            WritableUtils.writeVLong(out, fragmentCopy.getSize());
            List<String> ownerNodeNames = fragmentCopy.getOwnerNodeNames();
            WritableUtils.writeStringArray(out, ownerNodeNames.toArray(new String[ownerNodeNames.size()]));
            Map<String, String> attributes = fragmentCopy.getAttributes();
            WritableUtils.writeVInt(out, attributes.size());
            for (Map.Entry<String, String> entry : attributes.entrySet()) {
                WritableUtils.writeString(out, entry.getKey());
                WritableUtils.writeString(out, entry.getValue());
            }

View Full Code Here

                    String key = WritableUtils.readString(in);
                    String value = WritableUtils.readString(in);
                    attributes.put(key, value);
                }
            }
            this.fragment = new DirectInputFragment(path, offset, length, Arrays.asList(locations), attributes);


            try {
                Class<? extends DataFormat<?>> formatClass = (Class<? extends DataFormat<?>>) conf
                        .getClassByName(supportTypeName)
                        .asSubclass(DataFormat.class);

View Full Code Here

        }
        return results;
    }


    private static FileSplit getSplit(BlockMap blockMap, Path path, long start, long end) {
        DirectInputFragment f = blockMap.get(start, end);
        List<String> owners = f.getOwnerNodeNames();
        FileSplit split = new FileSplit(
                path, start, end - start,
                owners.toArray(new String[owners.size()]));
        return split;
    }

View Full Code Here

TOP

Related Classes of com.asakusafw.runtime.directio.DirectInputFragment

com.asakusafw.directio.hive.orc.AbstractOrcFileFormat

com.asakusafw.directio.hive.orc.OrcFileFormatTest

com.asakusafw.directio.hive.parquet.AbstractParquetFileFormat

com.asakusafw.directio.hive.parquet.ParquetFileFormatTest

com.asakusafw.runtime.directio.hadoop.BlockMap

com.asakusafw.runtime.stage.input.BridgeInputFormat$BridgeInputSplit

com.asakusafw.runtime.stage.input.TemporaryInputFormat

com.asakusafw.testdriver.directio.DirectIoTestHelper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.