Package org.apache.oodt.cas.pushpull.config

Examples of org.apache.oodt.cas.pushpull.config.DownloadInfo

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
        RemoteSite remoteSite = null;

        // parse property file
        VirtualFileStructure vfs = propFileParser.parse(new FileInputStream(
                propFile));
        DownloadInfo di = dfi.getDownloadInfo();
        if (!di.isAllowAliasOverride()
                || (remoteSite = vfs.getRemoteSite()) == null)
            remoteSite = di.getRemoteSite();
        LinkedList<String> fileList = FileRestrictions.toStringList(vfs
                .getRootVirtualFile());

        // download data files specified in property file
        for (String file : fileList) {
            try {
                linker.addPropFileToDataFileLink(propFile, file);
                if (!frs.addToDownloadQueue(remoteSite, file, di
                        .getRenamingConv(), di.getStagingArea(), dfi
                        .getQueryMetadataElementName(), di.deleteFromServer()))
                    linker.eraseLinks(propFile);
            } catch (ToManyFailedDownloadsException e) {
                throw new RetrievalMethodException(
                        "Connection appears to be down. . .unusual number of download failures. . .stopping : "
                                + e.getMessage());
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
        // parse property file
        VirtualFileStructure vfs = propFileParser.parse(new FileInputStream(
                propFile));

        // determine RemoteSite
        DownloadInfo di = dfi.getDownloadInfo();
        if (!di.isAllowAliasOverride()
                || (remoteSite = vfs.getRemoteSite()) == null)
            remoteSite = di.getRemoteSite();

        // modify vfs to be root based if HOME directory based
        if (!vfs.isRootBased()) {
            String homeDirPath = frs.getHomeDir(remoteSite).getProtocolPath()
                    .getPathString();
            VirtualFile root = new VirtualFile(homeDirPath, true);
            root.addChild(vfs.getRootVirtualFile());
            vfs = new VirtualFileStructure(homeDirPath + "/"
                    + vfs.getPathToRoot(), root.getRootDir());
            frs.changeToHOME(remoteSite);
        }

        // initialize variables
        final String initialCdPath = vfs.getPathToRoot();
        final VirtualFile vf = vfs.getRootVirtualFile();

        // change to initial directory (takes care of Linux auto-mounting)
        frs.changeToDir(initialCdPath, remoteSite);

        // add starting directory to stack
        Stack<ProtocolFile> files = new Stack<ProtocolFile>();
        files.add(frs.getCurrentFile(remoteSite));

        // start crawling
        while (!files.isEmpty()) {
            ProtocolFile file = files.peek();
            try {
                // if directory, then add its children to the crawl list
                if (file.isDirectory()) {

                    // get next page worth of children
                    List<ProtocolFile> children = frs.getNextPage(file,
                            new ProtocolFileFilter() {
                                public boolean accept(ProtocolFile pFile) {
                                    return FileRestrictions.isAllowed(pFile
                                            .getProtocolPath(), vf);
                                }
                            });

                    // if directory had more children then add them
                    if (children.size() > 0)
                        files.addAll(children);
                    // otherwise remove the directory from the crawl list
                    else
                        files.pop();

                    // if file, then download it
                } else {
                    linker.addPropFileToDataFileLink(propFile, file);
                    if (!frs.addToDownloadQueue(files.pop(), di
                            .getRenamingConv(), di.getStagingArea(), dfi
                            .getQueryMetadataElementName(), di
                            .deleteFromServer()))
                        linker.eraseLinks(propFile);
                }

            } catch (ToManyFailedDownloadsException e) {
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        // parse property file
        VirtualFileStructure vfs = propFileParser.parse(new FileInputStream(
                propFile));

        // determine RemoteSite
        DownloadInfo di = dfi.getDownloadInfo();
        if (!di.isAllowAliasOverride()
                || (remoteSite = vfs.getRemoteSite()) == null)
            remoteSite = di.getRemoteSite();

        // modify vfs to be root based if HOME directory based
        if (!vfs.isRootBased()) {
            String homeDirPath = frs.getHomeDir(remoteSite).getPath();
            VirtualFile root = new VirtualFile(homeDirPath, true);
            root.addChild(vfs.getRootVirtualFile());
            vfs = new VirtualFileStructure(homeDirPath + "/"
                    + vfs.getPathToRoot(), root.getRootDir());
            frs.changeToHOME(remoteSite);
        }

        // initialize variables
        final String initialCdPath = vfs.getPathToRoot();
        final VirtualFile vf = vfs.getRootVirtualFile();

        // change to initial directory (takes care of Linux auto-mounting)
        frs.changeToDir(initialCdPath, remoteSite);

        // add starting directory to stack
        Stack<RemoteSiteFile> files = new Stack<RemoteSiteFile>();
        files.add(new RemoteSiteFile(frs.getCurrentFile(remoteSite), remoteSite));

        // start crawling
        while (!files.isEmpty()) {
            RemoteSiteFile file = files.peek();
            try {
                // if directory, then add its children to the crawl list
                if (file.isDir()) {

                    // get next page worth of children
                    List<RemoteSiteFile> children = frs.getNextPage(file,
                            new ProtocolFileFilter() {
                                @Override
                              public boolean accept(ProtocolFile pFile) {
                                    return FileRestrictions.isAllowed(new
                                        ProtocolPath(pFile
                                            .getPath(), pFile.isDir()), vf);
                                }
                            });

                    // if directory had more children then add them
                    if (children.size() > 0)
                        files.addAll(children);
                    // otherwise remove the directory from the crawl list
                    else
                        files.pop();

                    // if file, then download it
                } else {
                    linker.addPropFileToDataFileLink(propFile, file);
                    if (!frs.addToDownloadQueue(files.pop(), di
                            .getRenamingConv(), di.getStagingArea(), dfi
                            .getQueryMetadataElementName(), di
                            .deleteFromServer()))
                        linker.eraseLinks(propFile);
                }

            } catch (ToManyFailedDownloadsException e) {
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
        // parse property file
        VirtualFileStructure vfs = propFileParser.parse(new FileInputStream(
                propFile));

        // determine RemoteSite
        DownloadInfo di = dfi.getDownloadInfo();
        if (!di.isAllowAliasOverride()
                || (remoteSite = vfs.getRemoteSite()) == null)
            remoteSite = di.getRemoteSite();

        // modify vfs to be root based if HOME directory based
        if (!vfs.isRootBased()) {
            String homeDirPath = frs.getHomeDir(remoteSite).getPath();
            VirtualFile root = new VirtualFile(homeDirPath, true);
            root.addChild(vfs.getRootVirtualFile());
            vfs = new VirtualFileStructure(homeDirPath + "/"
                    + vfs.getPathToRoot(), root.getRootDir());
            frs.changeToHOME(remoteSite);
        }

        // initialize variables
        final String initialCdPath = vfs.getPathToRoot();
        final VirtualFile vf = vfs.getRootVirtualFile();

        // change to initial directory (takes care of Linux auto-mounting)
        frs.changeToDir(initialCdPath, remoteSite);

        // add starting directory to stack
        Stack<RemoteSiteFile> files = new Stack<RemoteSiteFile>();
        files.add(new RemoteSiteFile(frs.getCurrentFile(remoteSite)));

        // start crawling
        while (!files.isEmpty()) {
            RemoteSiteFile file = files.peek();
            try {
                // if directory, then add its children to the crawl list
                if (file.isDir()) {

                    // get next page worth of children
                    List<RemoteSiteFile> children = frs.getNextPage(file,
                            new ProtocolFileFilter() {
                                public boolean accept(ProtocolFile pFile) {
                                    return FileRestrictions.isAllowed(new
                                        ProtocolPath(pFile
                                            .getPath(), pFile.isDir()), vf);
                                }
                            });

                    // if directory had more children then add them
                    if (children.size() > 0)
                        files.addAll(children);
                    // otherwise remove the directory from the crawl list
                    else
                        files.pop();

                    // if file, then download it
                } else {
                    linker.addPropFileToDataFileLink(propFile, file);
                    if (!frs.addToDownloadQueue(files.pop(), di
                            .getRenamingConv(), di.getStagingArea(), dfi
                            .getQueryMetadataElementName(), di
                            .deleteFromServer()))
                        linker.eraseLinks(propFile);
                }

            } catch (ToManyFailedDownloadsException e) {
View Full Code Here
TOP

Related Classes of org.apache.oodt.cas.pushpull.config.DownloadInfo

  • org.apache.oodt.cas.pushpull.retrievalmethod.ListRetriever
  • org.apache.oodt.cas.pushpull.retrievalmethod.RemoteCrawler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.