Package com.cloudera.recordbreaker.analyzer

Examples of com.cloudera.recordbreaker.analyzer.FileSummary


            item.add(new ExternalLink("schemalabellink", schemaUrl, schemalabel.toString()));

            List<TypeGuessSummary> typeGuesses = ss.getTypeGuesses();
            for (int i = 0; i < Math.min(1, typeGuesses.size()); i++) {
              TypeGuessSummary curTGS = typeGuesses.get(i);
              FileSummary fs = curTGS.getFileSummary();

              PageParameters pars = new PageParameters();
              String fidUrl = urlFor(FilePage.class, new PageParameters("fid=" + fs.getFid())).toString();
              item.add(new ExternalLink("schemafilelink", fidUrl, fs.getFname()));
            }
          }
        };
        add(new Label("numFisheyeSchemas", "" + list.size()));       
        add(listview);
View Full Code Here


            add(new Label("typecount", "" + tgses.size()));

            ListView<TypeGuessSummary> observationList = new ListView<TypeGuessSummary>("observations", tgses) {
              protected void populateItem(ListItem<TypeGuessSummary> item) {
                TypeGuessSummary tgs = item.getModelObject();
                FileSummary fs = tgs.getFileSummary();
                SchemaSummary ss = tgs.getSchemaSummary();

                String fileUrl = urlFor(FilePage.class, new PageParameters("fid=" + fs.getFid())).toString();
                item.add(new ExternalLink("filelink", fileUrl, fs.getFname()));

                String schemaUrl = urlFor(SchemaPage.class, new PageParameters("schemaid=" + ss.getSchemaId())).toString();
                item.add(new ExternalLink("schemalink", schemaUrl, "Schema"));
              }
            };
View Full Code Here

      if (fe.hasFSAndCrawl()) {
        if (fid1Str != null && fid2Str != null) {
          try {
            long fid1 = Long.parseLong(fid1Str);
            long fid2 = Long.parseLong(fid2Str);
            FileSummary fs1 = new FileSummary(fsa, fid1);
            FileSummary fs2 = new FileSummary(fsa, fid2);
            FileSummaryData fsd1 = fsa.getFileSummaryData(fid1);
            FileSummaryData fsd2 = fsa.getFileSummaryData(fid2);
            DataDescriptor dd1 = fsd1.getDataDescriptor();
            DataDescriptor dd2 = fsd2.getDataDescriptor();
            List<SchemaDescriptor> sds1 = dd1.getSchemaDescriptor();
            List<SchemaDescriptor> sds2 = dd2.getSchemaDescriptor();                       
            SchemaDescriptor sd1 = sds1.get(0);
            SchemaDescriptor sd2 = sds2.get(0);

            List<Schema> unionFreeSchemas1 = SchemaUtils.getUnionFreeSchemasByFrequency(sd1, 100, true);
            Schema schema1 = unionFreeSchemas1.get(0);
            List<Schema> unionFreeSchemas2 = SchemaUtils.getUnionFreeSchemasByFrequency(sd2, 100, true);
            Schema schema2 = unionFreeSchemas2.get(0);
           
            List<String> f1Attrs = new ArrayList<String>();
            List<String> f2Attrs = new ArrayList<String>();

            for (Schema.Field f: schema1.getFields()) {
              f1Attrs.add(f.name());
            }
            for (Schema.Field f: schema2.getFields()) {
              f2Attrs.add(f.name());
            }
           
            add(new ExternalLink("file1subtitlelink", urlFor(FilePage.class, new PageParameters("fid=" + fid1)).toString(), fs1.getPath().toString()));
            add(new ExternalLink("file2subtitlelink", urlFor(FilePage.class, new PageParameters("fid=" + fid2)).toString(), fs2.getPath().toString()));         
           
            add(new Label("file1title", fs1.getFname()));
            add(new Label("file2title", fs2.getFname()));

            add(new ListView<String>("fieldlist1", f1Attrs) {
                protected void populateItem(ListItem<String> listItem) {
                  String modelObj = listItem.getModelObject();
                  listItem.add(new Label("alabel", modelObj));
View Full Code Here

  public void onConfigure() {
    FishEye fe = FishEye.getInstance();
    AccessController accessCtrl = fe.getAccessController();
    FSAnalyzer fsAnalyzer = fe.getAnalyzer();
   
    FileSummary fileSummary = null;
    if (targetFid >= 0) {
      fileSummary = new FileSummary(fsAnalyzer, targetFid);
    } else if (targetName != null) {
      fileSummary = fsAnalyzer.getSingleFileSummary(targetName);
    }

    if (fileSummary != null) {
View Full Code Here

      if (fe.hasFSAndCrawl()) {
        if (fidStr != null) {
          try {
            this.fid = Long.parseLong(fidStr);
            final FileSummary fs = new FileSummary(fe.getAnalyzer(), fid);
            final long fsFid = fid;
            final String fsPath = fs.getPath().toString();
           
            FSAnalyzer fsa = fe.getAnalyzer();
            FileSummaryData fsd = fsa.getFileSummaryData(fid);
            DataDescriptor dd = fsd.getDataDescriptor();
            List<TypeGuessSummary> tgses = fs.getTypeGuesses();
           
            add(new Label("filetitle", fs.getFname()));
            add(new ExternalLink("filesubtitlelink", urlFor(FilesPage.class, new PageParameters("targetdir=" + fs.getPath().getParent().toString())).toString(), fs.getPath().getParent().toString()));
            // Set up the download file link
            IResourceStream hadoopfsStream = new AbstractResourceStreamWriter() {
                public void write(Response output) {
                  WebResponse weboutput = (WebResponse) output;
                  try {
                    BufferedInputStream in = new BufferedInputStream(new FileSummary(FishEye.getInstance().getAnalyzer(), fid).getRawBytes());
                    try {
                      byte buf[] = new byte[4096];
                      int contentLen = -1;
                      while ((contentLen = in.read(buf)) >= 0) {
                        weboutput.write(buf, 0, contentLen);
                      }
                    } finally {
                      try {
                        in.close();
                      } catch (IOException iex) {
                      }
                    }
                  } catch (IOException iex) {
                    iex.printStackTrace();
                  }
                }
                public Bytes length() {
                  return Bytes.bytes(new FileSummary(FishEye.getInstance().getAnalyzer(), fid).getSize());
                }
              };
            ResourceStreamResource resourceStream = new ResourceStreamResource(hadoopfsStream);
            resourceStream.setContentDisposition(ContentDisposition.ATTACHMENT);
            resourceStream.setFileName(fs.getFname());
            add(new ResourceLink<File>("downloadlink", resourceStream));

            // querySupported container holds queryform
            // queryUnsupported container holds an error message
            final boolean querySupported = dd.isHiveSupported() && fe.isQueryServerAvailable(false);
            final boolean hasJoins = joinPairs.size() > 0;
            add(new WebMarkupContainer("querySupported") {
                {
                  setOutputMarkupPlaceholderTag(true);
                  setVisibilityAllowed(querySupported);
                  add(new QueryForm("queryform", new ValueMap(), fid));
                  history.visitNewPage(fsFid, fsPath);

                  // Add support for join-choices here
                  add(new WebMarkupContainer("hasJoins") {
                      {
                        setOutputMarkupPlaceholderTag(true);
                        setVisibilityAllowed(hasJoins);
                       
                        add(new ListView<JoinPair>("joinlistview", joinPairs) {
                            protected void populateItem(ListItem<JoinPair> joinItem) {
                              JoinPair modelObj = joinItem.getModelObject();

                              PageParameters pps = new PageParameters();
                              pps.add("fid1", fidStr);
                              pps.add("fid2", "" + modelObj.getJoinFid());
                              joinItem.add(new ExternalLink("joinpath",
                                                            urlFor(JoinPage.class, pps).toString(),
                                                            modelObj.getJoinPath()));
                              ///joinItem.add(new Label("schemadesc", modelObj.getSchemaDesc()));
                            }
                          });
                      }
                    });
                }
              });
            add(new WebMarkupContainer("queryUnsupported") {
                {
                  setOutputMarkupPlaceholderTag(true);
                  setVisibilityAllowed(! querySupported);
                }
              });
           
            // File metadata
            add(new Label("owner", fs.getOwner()));
            add(new Label("size", "" + fs.getSize()));
            add(new Label("lastmodified", fs.getLastModified()));
            add(new Label("crawledon", fs.getCrawl().getStartedDate()));

            // Schema data
            if (tgses.size() > 0) {
              TypeGuessSummary tgs = tgses.get(0);
              TypeSummary ts = tgs.getTypeSummary();         
View Full Code Here

      setOutputMarkupPlaceholderTag(true);
      setVisibilityAllowed(false);
    }
    public void onConfigure() {
      FishEye fe = FishEye.getInstance();
      FileSummary fs = new FileSummary(fe.getAnalyzer(), fid);     
      AccessController accessCtrl = fe.getAccessController();
      setVisibilityAllowed(fe.hasFSAndCrawl() && (fs != null && accessCtrl.hasReadAccess(fs)));
    }
View Full Code Here

        // Single table query!
        ///
        if (fidStr != null) {
          try {
            fid = Long.parseLong(fidStr);
            FileSummary fs = new FileSummary(fe.getAnalyzer(), fid);
            DataQuery dq = DataQuery.getInstance();
            FSAnalyzer fsa = fe.getAnalyzer();
            FileSummaryData fsd = fsa.getFileSummaryData(fid);
            DataDescriptor dd = fsd.getDataDescriptor();

            if (dq != null) {
              queryResults = dq.query(dd, null, projClauseStr, selClauseStr);
            }
          } catch (Exception ex) {
            ex.printStackTrace();
          }
        }
        ///
        // Multi table query!
        ///
        if (fidStr1 != null) {
          try {
            fid1 = Long.parseLong(fidStr1);
            FileSummary fs1 = new FileSummary(fe.getAnalyzer(), fid1);
            fid2 = Long.parseLong(fidStr2);
            FileSummary fs2 = new FileSummary(fe.getAnalyzer(), fid2);
            DataQuery dq = DataQuery.getInstance();
            FSAnalyzer fsa = fe.getAnalyzer();
            FileSummaryData fsd1 = fsa.getFileSummaryData(fid1);
            DataDescriptor dd1 = fsd1.getDataDescriptor();
            FileSummaryData fsd2 = fsa.getFileSummaryData(fid2);
View Full Code Here

    public void onConfigure() {
      FishEye fe = FishEye.getInstance();
      AccessController accessCtrl = fe.getAccessController();
      if (fidStr != null) {
        FileSummary fs = new FileSummary(fe.getAnalyzer(), fid);
        setVisibilityAllowed(fe.hasFSAndCrawl() && (fs != null && accessCtrl.hasReadAccess(fs)));
      } else if (fidStr1 != null) {
        FileSummary fs1 = new FileSummary(fe.getAnalyzer(), fid1);
        FileSummary fs2 = new FileSummary(fe.getAnalyzer(), fid2);     
        setVisibilityAllowed(fe.hasFSAndCrawl() && (fs1 != null && accessCtrl.hasReadAccess(fs1)) && (fs2 != null && accessCtrl.hasReadAccess(fs2)));
      }
    }
View Full Code Here

      setVisibilityAllowed(false);
    } else {
      FishEye fe = FishEye.getInstance();
      AccessController accessCtrl = fe.getAccessController();   
      FSAnalyzer fsAnalyzer = fe.getAnalyzer();
      FileSummary fileSummary = new FileSummary(fsAnalyzer, fid);
      try {
        setVisibilityAllowed(fe.hasFSAndCrawl() && accessCtrl.hasReadAccess(fileSummary) && fileSummary.isDir() && fileSummary.getDataDescriptor().getSchemaDescriptor().size() > 0);
      } catch (IOException iex) {
        setVisibilityAllowed(false);
      }
    }
  }
View Full Code Here

        // I. Generate list of parent dirs for this directory
        //
        List<DirLabelPair> parentDirPairList = new ArrayList<DirLabelPair>();
        List<FileSummary> parentDirList = fe.getDirParents(targetDir);
       
        FileSummary lastDir = null;
        if (parentDirList != null) {
          for (FileSummary curDirSummary: parentDirList) {
            Path curDir = curDirSummary.getPath();
            String prefix = "";
            if (lastDir != null) {
              prefix = lastDir.getPath().toString();
            }
            String label = curDir.toString().substring(prefix.length());

            // Check rights
            if (accessCtrl.hasReadAccess(curDirSummary)) {
              String dirUrl = urlFor(FilesPage.class, new PageParameters("targetdir=" + curDir.toString())).toString();
              label = "<a href=\"" + dirUrl + "\">" + label + "</a>";
            }
            parentDirPairList.add(new DirLabelPair(label, curDir));
            lastDir = curDirSummary;
          }
        }
        String targetDirLabel = targetDir.substring(lastDir == null ? 0 : lastDir.getPath().toString().length());
        parentDirPairList.add(new DirLabelPair(targetDirLabel, new Path(targetDir)));

        add(new Label("lastParentDirEntry", parentDirPairList.get(parentDirPairList.size()-1).getLabel()));
        parentDirPairList.remove(parentDirPairList.size()-1);

        add(new ListView<DirLabelPair>("parentdirlisting", parentDirPairList) {
            protected void populateItem(ListItem<DirLabelPair> item) {
              DirLabelPair pair = item.getModelObject();
              item.add(new Label("dirlabel", pair.getLabel()).setEscapeModelStrings(false));
            }
          });

        //
        // II. Generate list of subdirs in the directory
        //
        final List<DirLabelPair> childDirPairList = new ArrayList<DirLabelPair>();
        List<FileSummary> childDirList = fe.getDirChildren(targetDir);
        if (childDirList != null) {
          for (FileSummary curDirSummary: childDirList) {
            String label = curDirSummary.getFname();
            if (accessCtrl.hasReadAccess(curDirSummary)) {
              String dirUrl = urlFor(FilesPage.class, new PageParameters("targetdir=" + curDirSummary.getPath().toString())).toString();
              label = "<a href=\"" + dirUrl + "\">" + label + "</a>";
            }
            childDirPairList.add(new DirLabelPair(label, curDirSummary.getPath()));
          }
        }
        add(new WebMarkupContainer("subdirbox") {
            {
              add(new ListView<DirLabelPair>("childdirlisting", childDirPairList) {
                  protected void populateItem(ListItem<DirLabelPair> item) {
                    DirLabelPair pair = item.getModelObject();
                    item.add(new Label("childdirlabel", pair.getLabel()).setEscapeModelStrings(false));
                  }
                });
              setOutputMarkupPlaceholderTag(true);
              setVisibilityAllowed(false);
            }
            public void onConfigure() {
              setVisibilityAllowed(childDirPairList.size() > 0);
            }
          });

        //
        // III. Generate list of files in the directory
        //
        List<FileSummary> filelist = FishEye.getInstance().getAnalyzer().getPrecachedFileSummariesInDir(false, targetDir);
        add(new Label("numFisheyeFiles", "" + filelist.size()));
        add(new ListView<FileSummary>("filelisting", filelist) {
            protected void populateItem(ListItem<FileSummary> item) {
              long start = System.currentTimeMillis();
              FileSummary fs = item.getModelObject();

              // 1.  Filename.  Link is conditional on having read access
              if (accessCtrl.hasReadAccess(fs)) {
                String fileUrl = urlFor(FilePage.class, new PageParameters("fid=" + fs.getFid())).toString();               
                item.add(new Label("filelabel", "<a href=\"" + fileUrl + "\">" + fs.getFname() + "</a>").setEscapeModelStrings(false));
              } else {
                item.add(new Label("filelabel", fs.getFname()));
              }

              // 2-5. A bunch of fields that get added no matter what the user's access rights.
              item.add(new Label("sizelabel", "" + fs.getSize()));
              item.add(new Label("ownerlabel", fs.getOwner()));
              item.add(new Label("grouplabel", fs.getGroup()));
              item.add(new Label("permissionslabel", fs.getPermissions().toString()));

              // 6-7.  Fields that have link conditional on read access AND the existence of relevant info.
              if (accessCtrl.hasReadAccess(fs)) {
                List<TypeGuessSummary> tgs = fs.getTypeGuesses();
                if (tgs.size() > 0) {
                  TypeSummary ts = tgs.get(0).getTypeSummary();
                  SchemaSummary ss = tgs.get(0).getSchemaSummary();

                  String typeUrl = urlFor(FiletypePage.class, new PageParameters("typeid=" + ts.getTypeId())).toString();
View Full Code Here

TOP

Related Classes of com.cloudera.recordbreaker.analyzer.FileSummary

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.