Examples of com.dotcms.repackage.com.csvreader.CsvReader

Package com.dotcms.repackage.com.csvreader

Examples of com.dotcms.repackage.com.csvreader.CsvReader

com.dotcms.repackage.com.csvreader.CsvReader


    public Boolean checkFoldersIntegrity(String endpointId) throws Exception {


        try {


            CsvReader folders = new CsvReader(ConfigUtils.getIntegrityPath() + File.separator + endpointId + File.separator + IntegrityType.FOLDERS.getDataToCheckCSVName(), '|');
            boolean tempCreated = false;
            DotConnect dc = new DotConnect();
            String tempTableName = getTempTableName(endpointId, IntegrityType.FOLDERS);


            // lets create a temp table and insert all the records coming from the CSV file
            String tempKeyword = getTempKeyword();


            String createTempTable = "create " +tempKeyword+ " table " + tempTableName + " (inode varchar(36) not null, identifier varchar(36) not null,parent_path varchar(255), "
                    + "asset_name varchar(255), host_identifier varchar(36) not null, primary key (inode) )" + (DbConnectionFactory.isOracle()?" ON COMMIT PRESERVE ROWS ":"");


            if(DbConnectionFactory.isOracle()) {
                createTempTable=createTempTable.replaceAll("varchar\\(", "varchar2\\(");
            }


            final String INSERT_TEMP_TABLE = "insert into " + tempTableName + " values(?,?,?,?,?)";


            while (folders.readRecord()) {


                if(!tempCreated) {
                    dc.executeStatement(createTempTable);
                    tempCreated = true;
                }


                String folderInode = folders.get(0);
        String folderIdentifier = folders.get(1);
        String parentPath = folders.get(2);
        String assetName = folders.get(3);
        String hostIdentifier = folders.get(4);


        dc.setSQL(INSERT_TEMP_TABLE);
        dc.addParam(folderInode);
        dc.addParam(folderIdentifier);
        dc.addParam(parentPath);
        dc.addParam(assetName);
        dc.addParam(hostIdentifier);
        dc.loadResult();
            }


            folders.close();


            String resultsTableName = getResultsTableName(IntegrityType.FOLDERS);


            // compare the data from the CSV to the local db data and see if we have conflicts
            dc.setSQL("select 1 from identifier iden "

View Full Code Here


    public Boolean checkStructuresIntegrity(String endpointId) throws Exception {


        try {


            CsvReader structures = new CsvReader(ConfigUtils.getIntegrityPath() + File.separator + endpointId + File.separator + IntegrityType.STRUCTURES.getDataToCheckCSVName(), '|');
            boolean tempCreated = false;
            DotConnect dc = new DotConnect();
            String tempTableName = getTempTableName(endpointId, IntegrityType.STRUCTURES);


            String tempKeyword = getTempKeyword();


            String createTempTable = "create " +tempKeyword+ " table " + tempTableName + " (inode varchar(36) not null, velocity_var_name varchar(255), "
                    + " primary key (inode) )";


            if(DbConnectionFactory.isOracle()) {
                createTempTable=createTempTable.replaceAll("varchar\\(", "varchar2\\(");
            }


            final String INSERT_TEMP_TABLE = "insert into " + tempTableName + " values(?,?)";


            while (structures.readRecord()) {


                if(!tempCreated) {
                    dc.executeStatement(createTempTable);
                    tempCreated = true;
                }


                //select f.inode, i.parent_path, i.asset_name, i.host_inode
                String structureInode = structures.get(0);
                String verVarName = structures.get(1);


                dc.setSQL(INSERT_TEMP_TABLE);
                dc.addParam(structureInode);
                dc.addParam(verVarName);
                dc.loadResult();
            }


            structures.close();


            String resultsTableName = getResultsTableName(IntegrityType.STRUCTURES);


            // compare the data from the CSV to the local db data and see if we have conflicts
            dc.setSQL("select s.velocity_var_name as velocity_name, "

View Full Code Here


    public Boolean checkWorkflowSchemesIntegrity(String endpointId) throws Exception {


        try {


            CsvReader schemes = new CsvReader(ConfigUtils.getIntegrityPath() + File.separator + endpointId + File.separator + IntegrityType.SCHEMES.getDataToCheckCSVName(), '|');
            boolean tempCreated = false;
            DotConnect dc = new DotConnect();
            String tempTableName = getTempTableName(endpointId, IntegrityType.SCHEMES);


            String tempKeyword = getTempKeyword();


            String createTempTable = "create " +tempKeyword+ " table " + tempTableName + " (inode varchar(36) not null, name varchar(255), "
                    + " primary key (inode) )";


            if(DbConnectionFactory.isOracle()) {
                createTempTable=createTempTable.replaceAll("varchar\\(", "varchar2\\(");
            }


            final String INSERT_TEMP_TABLE = "insert into " + tempTableName + " values(?,?)";


            while (schemes.readRecord()) {


                if(!tempCreated) {
                    dc.executeStatement(createTempTable);
                    tempCreated = true;
                }


                //select f.inode, i.parent_path, i.asset_name, i.host_inode
                String schemeInode = schemes.get(0);
                String name = schemes.get(1);


                dc.setSQL(INSERT_TEMP_TABLE);
                dc.addParam(schemeInode);
                dc.addParam(name);
                dc.loadResult();
            }


            schemes.close();


            String resultsTableName = getResultsTableName(IntegrityType.SCHEMES);


            // compare the data from the CSV to the local db data and see if we have conflicts
            dc.setSQL("select s.name, s.id as local_inode, wt.inode as remote_inode from workflow_scheme s "

View Full Code Here


    public void generateDataToFixTable(String endpointId, IntegrityType type) throws Exception {


        try {


            CsvReader csvFile = new CsvReader(ConfigUtils.getIntegrityPath() + File.separator + endpointId + File.separator + type.getDataToFixCSVName(), '|');
            DotConnect dc = new DotConnect();
            String resultsTable = getResultsTableName(type);


            String INSERT_TEMP_TABLE = "insert into " + resultsTable + " (local_inode, remote_inode, endpoint_id) values(?,?,?)";


      if(type==IntegrityType.FOLDERS) {
        INSERT_TEMP_TABLE = "insert into " + resultsTable + " (local_inode, remote_inode, local_identifier, remote_identifier, endpoint_id) values(?,?,?,?,?)";
      }


      while (csvFile.readRecord()) {


        //select f.inode, i.parent_path, i.asset_name, i.host_inode
        String localInode = csvFile.get(0);
        String remoteInode = csvFile.get(1);
        dc.setSQL(INSERT_TEMP_TABLE);
        dc.addParam(localInode);
        dc.addParam(remoteInode);


        if(type==IntegrityType.FOLDERS) {
          String localIdentifier = csvFile.get(2);
          String remoteIdentifier = csvFile.get(3);
          dc.addParam(localIdentifier);
          dc.addParam(remoteIdentifier);
        }


        dc.addParam(endpointId);

View Full Code Here

        Reader reader = createTempFile( "Title, Host" + "\r\n" +
                "Test1, " + defaultHost.getIdentifier() + "\r\n" +
                "Test2, " + defaultHost.getIdentifier() + "\r\n" +
                "Test3, " + defaultHost.getIdentifier() + "\r\n" +
                "Test4, " + defaultHost.getIdentifier() + "\r\n" );
        CsvReader csvreader = new CsvReader( reader );
        csvreader.setSafetySwitch( false );
        String[] csvHeaders = csvreader.getHeaders();


        //Preview=true
        HashMap<String, List<String>> results = ImportUtil.importFile( 0L, defaultHost.getInode(), structure1.getInode(), new String[]{}, true, false, user, defaultLanguage.getId(), csvHeaders, csvreader, -1, -1, reader );
        //Validations
        validate( results, true, false, true );


        //As it was a preview nothing should be saved
        List<Contentlet> savedData = contentletAPI.findByStructure( structure1.getInode(), user, false, 0, 0 );
        //Validations
        assertNotNull( savedData );
        assertEquals( savedData.size(), 0 );


        //----------------PREVIEW = FALSE-----------------------------------------
        //------------------------------------------------------------------------
        //Create the csv file to import
        reader = createTempFile( "Title, Host" + "\r\n" +
                "Test1, " + defaultHost.getIdentifier() + "\r\n" +
                "Test2, " + defaultHost.getIdentifier() + "\r\n" +
                "Test3, " + defaultHost.getIdentifier() + "\r\n" +
                "Test4, " + defaultHost.getIdentifier() + "\r\n" );
        csvreader = new CsvReader( reader );
        csvreader.setSafetySwitch( false );
        csvHeaders = csvreader.getHeaders();


        //Preview=false
        results = ImportUtil.importFile( 0L, defaultHost.getInode(), structure1.getInode(), new String[]{}, false, false, user, defaultLanguage.getId(), csvHeaders, csvreader, -1, -1, reader );
        //Validations
        validate( results, false, false, true );


        //Now we should have saved data
        savedData = contentletAPI.findByStructure( structure1.getInode(), user, false, 0, 0 );
        //Validations
        assertNotNull( savedData );
        assertEquals( savedData.size(), 4 );


        //----------------USING WRONG HOST IDENTIFIERS----------------------------
        //------------------------------------------------------------------------
        //Create the csv file to import
        reader = createTempFile( "Title, Host" + "\r\n" +
                "Test5, " + defaultHost.getIdentifier() + "\r\n" +
                "Test6, " + "999-99999999-99999999-00000" + "\r\n" +
                "Test7, " + "44444444-5555555555-2222" + "\r\n" );
        csvreader = new CsvReader( reader );
        csvreader.setSafetySwitch( false );
        csvHeaders = csvreader.getHeaders();


        //Preview=true
        results = ImportUtil.importFile( 0L, defaultHost.getInode(), structure1.getInode(), new String[]{}, true, false, user, defaultLanguage.getId(), csvHeaders, csvreader, -1, -1, reader );
        //Validations
        validate( results, true, true, true );


        //We should have the same amount on data
        savedData = contentletAPI.findByStructure( structure1.getInode(), user, false, 0, 0 );
        //Validations
        assertNotNull( savedData );
        assertEquals( savedData.size(), 4 );


        //---------------USING KEY FIELDS-----------------------------------------
        //------------------------------------------------------------------------


        //Making sure the contentlets are in the indexes
        List<ContentletSearch> contentletSearchResults;
        int x = 0;
        do {
            Thread.sleep( 200 );
            //Verify if it was added to the index
            contentletSearchResults = contentletAPI.searchIndex( "+structureName:" + structure1.getVelocityVarName() + " +working:true +deleted:false +" + structure1.getVelocityVarName() + ".title:Test1 +languageId:1", 0, -1, null, user, true );
            x++;
        } while ( (contentletSearchResults == null || contentletSearchResults.isEmpty()) && x < 100 );


        //Create the csv file to import
        reader = createTempFile( "Title, Host" + "\r\n" +
                "Test1, " + defaultHost.getIdentifier() + "\r\n" +
                "Test2, " + defaultHost.getIdentifier() + "\r\n" );
        csvreader = new CsvReader( reader );
        csvreader.setSafetySwitch( false );
        csvHeaders = csvreader.getHeaders();


        //Preview=false
        results = ImportUtil.importFile( 0L, defaultHost.getInode(), structure1.getInode(), new String[]{textFileStructure1.getInode()}, false, false, user, defaultLanguage.getId(), csvHeaders, csvreader, -1, -1, reader );
        //Validations
        validate( results, false, false, true );//We should expect warnings: Line #X. The key fields chosen match 1 existing content(s) - more than one match suggests key(s) are not properly unique


        //We used the key fields, so the import process should update instead to add new records
        savedData = contentletAPI.findByStructure( structure1.getInode(), user, false, 0, 0 );
        //Validations
        assertNotNull( savedData );
        assertEquals( savedData.size(), 4 );


        //---------------USING IDENTIFIER COLUMN----------------------------------
        //------------------------------------------------------------------------
        //Create the csv file to import


        String id1 = null;
        String id2 = null;
        for ( Contentlet content : savedData ) {
            if ( content.getMap().get( "title" ).equals( "Test1" ) ) {
                id1 = content.getIdentifier();
            } else if ( content.getMap().get( "title" ).equals( "Test2" ) ) {
                id2 = content.getIdentifier();
            }
        }


        reader = createTempFile( "Identifier, Title, Host" + "\r\n" +
                id1 + ", Test1_edited, " + defaultHost.getIdentifier() + "\r\n" +
                id2 + ", Test2_edited, " + defaultHost.getIdentifier() + "\r\n" );
        csvreader = new CsvReader( reader );
        csvreader.setSafetySwitch( false );
        csvHeaders = csvreader.getHeaders();


        //Preview=false
        results = ImportUtil.importFile( 0L, defaultHost.getInode(), structure1.getInode(), new String[]{}, false, false, user, defaultLanguage.getId(), csvHeaders, csvreader, -1, -1, reader );
        //Validations
        validate( results, false, false, true );


        //We used a identifier column, so the import process should update instead to add new records
        savedData = contentletAPI.findByStructure( structure1.getInode(), user, false, 0, 0 );
        //Validations
        assertNotNull( savedData );
        assertEquals( savedData.size(), 4 );


        //-------------------------LANGUAGE AND KEY FIELDS------------------------
        //------------------------------------------------------------------------
        //Create the csv file to import
        reader = createTempFile( "languageCode, countryCode, Title, Host" + "\r\n" +
                "es, ES, Test1_edited, " + defaultHost.getIdentifier() + "\r\n" +
                "es, ES, Test2_edited, " + defaultHost.getIdentifier() + "\r\n" );
        csvreader = new CsvReader( reader );
        csvreader.setSafetySwitch( false );
        csvHeaders = csvreader.getHeaders();


        int languageCodeHeaderColumn = 0;
        int countryCodeHeaderColumn = 1;
        //Preview=false
        results = ImportUtil.importFile( 0L, defaultHost.getInode(), structure1.getInode(), new String[]{textFileStructure1.getInode()}, false, true, user, -1, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader );

View Full Code Here

      return 1;
    }
  }


  private void saveOrUpdateCat(String contextInode, BufferedReader br, Boolean merge) throws IOException, Exception {
    CsvReader csvreader = new CsvReader(br);
    csvreader.setSafetySwitch(false);
    csvreader.readHeaders();
    String[] csvLine;


    while (csvreader.readRecord()) {
      csvLine = csvreader.getValues();
      try {
        saveOrUpdateCategory(true, contextInode, csvLine[0], csvLine[2], csvLine[1], null, csvLine[3], merge);


      } catch(Exception e) {
        Logger.error(this, "Error trying to save/update the categories csv row: name=" +csvLine[0]+ ", variable=" + csvLine[2] + ", key=" + csvLine[1] + ", sort=" + csvLine[3] , e);
      }
    }


    csvreader.close();
    br.close();
  }

View Full Code Here

     * @return
     * @throws IOException
     */
    public static List<String[]> specialSplit(Reader reader, char delim, char textQualifier) throws IOException {
    List<String[]> records = new ArrayList<String[]>();
    CsvReader csvReader = new CsvReader (reader);
    csvReader.setDelimiter(delim);
    csvReader.setTextQualifier(textQualifier);
    csvReader.readHeaders();
    records.add(csvReader.getHeaders());
    while(csvReader.readRecord()) {
      records.add(csvReader.getValues());
    }
    return records;
    }

View Full Code Here


    //Parsing the file line per line
    Reader reader = null;
    try {
      reader = new InputStreamReader(new ByteArrayInputStream(bytes));
      CsvReader csvreader = new CsvReader(reader);
      csvreader.setSafetySwitch(false);


      if (csvreader.readHeaders()) {


        //Importing headers from the first file line
        importHeaders(csvreader.getHeaders(), st, keyfields, preview, user);
        lineNumber++;


        //Reading the whole file
        if (headers.size() > 0) {


          if (!preview)
            HibernateUtil.startTransaction();


          while (csvreader.readRecord()) {
            lineNumber++;
            try {
              lines++;
              Logger.debug(this, "Line " + lines + ": (" + csvreader.getRawRecord() + ").");
              //Importing a line
              importLine(csvreader.getValues(), st, preview, user, lineNumber);


              if (!preview && (lineNumber % commitGranularity == 0)) {
                HibernateUtil.commitTransaction();
                HibernateUtil.startTransaction();
              }


              if (!preview)
                Thread.sleep(sleepTime);
            } catch (DotRuntimeException ex) {
              String errorMessage = ex.getMessage();
              if(errorMessage.indexOf("Line #") == -1){
                errorMessage = "Line #"+lineNumber+" "+errorMessage;
              }
              myOutput.println(errorMessage);
              errors++;
              myOutput.println("Error line: " + lines + " (" + csvreader.getRawRecord()+ "). Line Ignored.");
            }
          }


          HibernateUtil.commitTransaction();

View Full Code Here

          SessionMessages.add(req, "error", "message.contentlet.file.required");
          setForward(req, "portlet.ext.contentlet.import_contentlets");
        } else {
          try {
            Reader reader = null;
            CsvReader csvreader = null;
            String[] csvHeaders = null;
            int languageCodeHeaderColumn = -1;
            int countryCodeHeaderColumn = -1;
                        
            if (importContentletsForm.getLanguage() == -1)
              reader = new InputStreamReader(new ByteArrayInputStream(bytes), Charset.forName("UTF-8"));
            else
              reader = new InputStreamReader(new ByteArrayInputStream(bytes));
            
            csvreader = new CsvReader(reader);
            csvreader.setSafetySwitch(false);
            
            switch ((int) importContentletsForm.getLanguage()) {
            case -1:
              if (0 < importContentletsForm.getFields().length) {
                if (csvreader.readHeaders()) {
                  csvHeaders = csvreader.getHeaders();
                  for (int column = 0; column < csvHeaders.length; ++column) {
                    if (csvHeaders[column].equals(languageCodeHeader))
                      languageCodeHeaderColumn = column;
                    if (csvHeaders[column].equals(countryCodeHeader))
                      countryCodeHeaderColumn = column;
                    
                    if ((-1 < languageCodeHeaderColumn) && (-1 < countryCodeHeaderColumn))
                      break;
                  }
                  
                  if ((-1 == languageCodeHeaderColumn) || (-1 == countryCodeHeaderColumn)) {
                    SessionMessages.add(req, "error", "message.import.contentlet.csv_headers.required");
                    setForward(req, "portlet.ext.contentlet.import_contentlets");
                  } else {
                    _generatePreview(0,req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
                    setForward(req, "portlet.ext.contentlet.import_contentlets_preview");
                  }
                } else {
                  SessionMessages.add(req, "error", "message.import.contentlet.csv_headers.error");
                  setForward(req, "portlet.ext.contentlet.import_contentlets");
                }
              } else {
                SessionMessages.add(req, "error", "message.import.contentlet.key_field.required");
                setForward(req, "portlet.ext.contentlet.import_contentlets");
              }
              break;
            case 0:
              SessionMessages.add(req, "error", "message.import.contentlet.language.required");
              setForward(req, "portlet.ext.contentlet.import_contentlets");
              break;
            default:
              _generatePreview(0, req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
              setForward(req, "portlet.ext.contentlet.import_contentlets_preview");
              break;
            }
            
            csvreader.close();
          } catch (Exception e) {
            _handleException(e, req);
            return;
          }
        }


      } catch (Exception ae) {
        _handleException(ae, req);
        return;
      }
    } else if ((cmd != null) && cmd.equals(com.dotmarketing.util.Constants.PUBLISH)) {
      AdminLogger.log(ImportContentletsAction.class, "processAction", "Importing Contentlets", user);
      Long importIdComplete=(Long)session.getAttribute("importId");
      String subcmd = req.getParameter("subcmd");
       if(subcmd != null && subcmd.equals("importContentletsResults")){
        HashMap<String, List<String>>results=ImportAuditUtil.loadImportResults(importIdComplete);
        req.setAttribute("importResults", results);
        setForward(req, "portlet.ext.contentlet.import_contentlets_results");
      } else {
        final ActionRequestImpl reqImpl = (ActionRequestImpl) req;
        final HttpServletRequest httpReq = reqImpl.getHttpServletRequest();
        final long importId = ImportAuditUtil.createAuditRecord(user.getUserId(), (String)httpReq.getSession().getAttribute("fileName"));
        Thread t=new Thread() {
          public void run() {
            try {
              Logger.debug(this, "Calling Process File Method");
              
              Reader reader = null;
              CsvReader csvreader = null;
              String[] csvHeaders = null;
              int languageCodeHeaderColumn = -1;
              int countryCodeHeaderColumn = -1;
              
              byte[] bytes = (byte[]) httpReq.getSession().getAttribute("file_to_import");
              File file= (File)httpReq.getSession().getAttribute("csvFile");
              ImportContentletsForm importContentletsForm = (ImportContentletsForm) form;
              if (importContentletsForm.getLanguage() == -1)
                reader = new InputStreamReader(new ByteArrayInputStream(bytes), Charset.forName("UTF-8"));
              else
                reader = new InputStreamReader(new ByteArrayInputStream(bytes));
              csvreader = new CsvReader(reader);
              csvreader.setSafetySwitch(false);
                
              if (importContentletsForm.getLanguage() == -1) {
                if (csvreader.readHeaders()) {
                  csvHeaders = csvreader.getHeaders();
                  for (int column = 0; column < csvHeaders.length; ++column) {
                    if (csvHeaders[column].equals(languageCodeHeader))
                      languageCodeHeaderColumn = column;
                    if (csvHeaders[column].equals(countryCodeHeader))
                      countryCodeHeaderColumn = column;
                      
                    if ((-1 < languageCodeHeaderColumn) && (-1 < countryCodeHeaderColumn))
                      break;
                  }
                }
              }
              HttpSession session = ((ActionRequestImpl)req).getHttpServletRequest().getSession();
              User user = _getUser(req);
              //long importId = ImportAuditUtil.createAuditRecord(user.getUserId(), (String)httpReq.getSession().getAttribute("fileName"));
              
      
              HashMap<String, List<String>> importresults=null;
              if(importSession.equals((String) session.getAttribute("importSession") )){
                session.removeAttribute("importSession");
                importresults=_processFile(importId, req, res, config, form, user, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
                
              }
                      
              List<String> counters= importresults .get("counters");
              int contentsToImport=0;
              for(String counter: counters ){
                String counterArray[]=counter.split("=");
                if(counterArray[0].equals("newContent") || counterArray[0].equals("contentToUpdate"))
                  contentsToImport=contentsToImport + Integer.parseInt(counterArray[1]);    
              }
              
              List<String> inodes= importresults.get("lastInode");
              if(!inodes.isEmpty()){
                ImportAuditUtil.updateAuditRecord(inodes.get(0), contentsToImport, importId,importresults);
              }
                      
              csvreader.close();
              
      
            } catch (Exception ae) {
              _handleException(ae, req);
              return;

View Full Code Here

TOP

Related Classes of com.dotcms.repackage.com.csvreader.CsvReader

com.dotcms.rest.IntegrityUtil

com.dotmarketing.portlets.categories.ajax.CategoryAjax

com.dotmarketing.portlets.contentlet.action.ImportContentletsAction

com.dotmarketing.quartz.job.ContentImportThread

com.dotmarketing.util.ImportUtilTest

com.dotmarketing.util.UtilMethods

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.