Examples of org.apache.commons.csv.CSVParser

org.apache.commons.csv.CSVParser
.ietf.org/html/rfc4180" target="_blank">RFC 4180 format.

To parse CSV input in a format like Excel, you write:
```
 CSVParser parser = CSVParser.parse(csvData, CSVFormat.EXCEL); for (CSVRecord csvRecord : parser) { ... } 
```
If the predefined formats don't match the format at hands, custom formats can be defined. More information about customising CSVFormats is available in {@link CSVFormat CSVFormat JavaDoc}.

Parsing into memory

If parsing record wise is not desired, the contents of the input can be read completely into memory.
```
 Reader in = new StringReader("a;b\nc;d"); CSVParser parser = new CSVParser(in, CSVFormat.EXCEL); List<CSVRecord> list = parser.getRecords(); 
```
There are two constraints that have to be kept in mind:
1. Parsing into memory starts at the current position of the parser. If you have already parsed records from the input, those records will not end up in the in memory representation of your CSV data.
2. Parsing into memory may consume a lot of system resources depending on the input. For example if you're parsing a 150MB file of CSV data the contents will be read completely into memory.
Notes

Internal parser state is completely covered by the format and the reader-state.
@version $Id: CSVParser.java 1617069 2014-08-10 08:53:42Z britter $ @see package documentation for more details

    }


    public Object unmarshal(Exchange exchange, InputStream inputStream) throws Exception {
        InputStreamReader in = new InputStreamReader(inputStream);
        try {
            CSVParser parser = new CSVParser(in, getStrategy());
            List<List<String>> list = new ArrayList<List<String>>();
            while (true) {
                String[] strings = parser.getLine();
                if (strings == null) {
                    break;
                }
                List<String> line = Arrays.asList(strings);
                list.add(line);

View Full Code Here

     *
     * @param fileName
     * @throws Exception
     */
    public void upsert(String fileName) throws Exception {
        CSVParser parser = CSVParser.parse(new File(fileName),
                format);
        upsert(parser);
    }

View Full Code Here

                format);
        upsert(parser);
    }


    public void upsert(Reader reader) throws Exception {
        CSVParser parser = new CSVParser(reader,format);
        upsert(parser);
    }

View Full Code Here


        public CSVRecord parse(String input) throws IOException {
            // TODO Creating a new parser for each line seems terribly inefficient but
            // there's no public way to parse single lines via commons-csv. We should update
            // it to create a LineParser class like this one.
            CSVParser csvParser = new CSVParser(new StringReader(input), csvFormat);
            return Iterables.getFirst(csvParser, null);
        }

View Full Code Here

      this.strategy = strategy;
      this.base = base;
    }


    void add(SolrInputDocument doc, int line, int column, String val) {
      CSVParser parser = new CSVParser(new StringReader(val), strategy);
      try {
        String[] vals = parser.getLine();
        if (vals!=null) {
          for (String v: vals) base.add(doc,line,column,v);
        } else {
          base.add(doc,line,column,val);
        }

View Full Code Here

      for (int i=0; i<skipLines; i++) {
        r.readLine();
      }
    }


    CSVParser parser = new CSVParser(reader, strategy);


    // parse the fieldnames from the header of the file
    if (fieldnames==null) {
      fieldnames = parser.getLine();
      if (fieldnames==null) {
        throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Expected fieldnames in CSV input");
      }
      prepareFields();
    }


    // read the rest of the CSV file
    for(;;) {
      int line = parser.getLineNumber();  // for error reporting in MT mode
      String[] vals = parser.getLine();
      if (vals==null) break;


      if (vals.length != fields.length) {
        input_err("expected "+fields.length+" values but got "+vals.length, vals, line);
      }

View Full Code Here

    }


    @Test
    public void testCSVCommonsUpsertBadEncapsulatedControlChars()
            throws Exception {
        CSVParser parser = null;
        PhoenixConnection conn = null;
        try {
            // Create table
            String statements = "CREATE TABLE IF NOT EXISTS "
                    + ENCAPSULATED_CHARS_TABLE
                    + "(MYKEY VARCHAR NOT NULL PRIMARY KEY, MYVALUE VARCHAR);";
            conn = DriverManager.getConnection(getUrl())
                    .unwrap(PhoenixConnection.class);
            PhoenixRuntime.executeStatements(conn,
                    new StringReader(statements), null);


            // Upsert CSV file
            CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn,
                    ENCAPSULATED_CHARS_TABLE, Collections.<String> emptyList(),
                    true);
            try {
                csvUtil.upsert(new StringReader(
                        CSV_VALUES_BAD_ENCAPSULATED_CONTROL_CHARS_WITH_HEADER));
                fail();
            } catch (RuntimeException e) {
                assertTrue(
                        e.getMessage(),
                        e.getMessage()
                                .contains(
                                        "invalid char between encapsulated token and delimiter"));
            }
        } finally {
            if (parser != null)
                parser.close();
            if (conn != null)
                conn.close();
        }
    }

View Full Code Here

        }
    }


    @Test
    public void testCSVCommonsUpsert_WithArray() throws Exception {
        CSVParser parser = null;
        PhoenixConnection conn = null;
        try {


            // Create table
            String statements = "CREATE TABLE IF NOT EXISTS ARRAY_TABLE "
                    + "(ID BIGINT NOT NULL PRIMARY KEY, VALARRAY INTEGER ARRAY);";
            conn = DriverManager.getConnection(getUrl()).unwrap(
                    PhoenixConnection.class);
            PhoenixRuntime.executeStatements(conn,
                    new StringReader(statements), null);


            // Upsert CSV file
            CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn, "ARRAY_TABLE",
                    null, true, ',', '"', null, "!");
            csvUtil.upsert(
                    new StringReader("ID,VALARRAY\n"
                            + "1,2!3!4\n"));


            // Compare Phoenix ResultSet with CSV file content
            PreparedStatement statement = conn
                    .prepareStatement("SELECT ID, VALARRAY FROM ARRAY_TABLE");
            ResultSet phoenixResultSet = statement.executeQuery();
            assertTrue(phoenixResultSet.next());
            assertEquals(1L, phoenixResultSet.getLong(1));
            assertEquals(
                    PArrayDataType.instantiatePhoenixArray(PDataType.INTEGER, new Integer[]{2, 3, 4}),
                    phoenixResultSet.getArray(2));
            assertFalse(phoenixResultSet.next());
        } finally {
            if (parser != null)
                parser.close();
            if (conn != null)
                conn.close();
        }
    }

View Full Code Here

            + "\n"
            + CSV_VALUES_BAD_ENCAPSULATED_CONTROL_CHARS;


    @Test
    public void testCSVCommonsUpsert() throws Exception {
        CSVParser parser = null;
        PhoenixConnection conn = null;
        try {


            // Create table
            String statements = "CREATE TABLE IF NOT EXISTS " + STOCK_TABLE
                    + "(SYMBOL VARCHAR NOT NULL PRIMARY KEY, COMPANY VARCHAR);";
            conn = DriverManager.getConnection(getUrl()).unwrap(
                    PhoenixConnection.class);
            PhoenixRuntime.executeStatements(conn,
                    new StringReader(statements), null);


            // Upsert CSV file
            CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn, STOCK_TABLE,
                    Collections.<String> emptyList(), true);
            csvUtil.upsert(new StringReader(STOCK_CSV_VALUES_WITH_HEADER));


            // Compare Phoenix ResultSet with CSV file content
            PreparedStatement statement = conn
                    .prepareStatement("SELECT SYMBOL, COMPANY FROM "
                            + STOCK_TABLE);
            ResultSet phoenixResultSet = statement.executeQuery();
            parser = new CSVParser(new StringReader(
                    STOCK_CSV_VALUES_WITH_HEADER), csvUtil.getFormat());
            for (CSVRecord record : parser) {
                assertTrue(phoenixResultSet.next());
                int i = 0;
                for (String value : record) {
                    assertEquals(value, phoenixResultSet.getString(i + 1));
                    i++;
                }
            }
            assertFalse(phoenixResultSet.next());
        } finally {
            if (parser != null)
                parser.close();
            if (conn != null)
                conn.close();
        }
    }

View Full Code Here

        }
    }


    @Test
    public void testTDVCommonsUpsert() throws Exception {
        CSVParser parser = null;
        PhoenixConnection conn = null;
        try {


            // Create table
            String statements = "CREATE TABLE IF NOT EXISTS " + STOCK_TABLE
                    + "(SYMBOL VARCHAR NOT NULL PRIMARY KEY, COMPANY VARCHAR);";
            conn = DriverManager.getConnection(getUrl()).unwrap(
                    PhoenixConnection.class);
            PhoenixRuntime.executeStatements(conn,
                    new StringReader(statements), null);


            // Upsert TDV file
            CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn, STOCK_TABLE,Collections.<String> emptyList()
                    , true, '\t', '"', null, CSVCommonsLoader.DEFAULT_ARRAY_ELEMENT_SEPARATOR);
            csvUtil.upsert(new StringReader(STOCK_TDV_VALUES_WITH_HEADER));


            // Compare Phoenix ResultSet with CSV file content
            PreparedStatement statement = conn
                    .prepareStatement("SELECT SYMBOL, COMPANY FROM "
                            + STOCK_TABLE);
            ResultSet phoenixResultSet = statement.executeQuery();
            parser = new CSVParser(new StringReader(
                    STOCK_TDV_VALUES_WITH_HEADER), csvUtil.getFormat());
            for (CSVRecord record : parser) {
                assertTrue(phoenixResultSet.next());
                int i = 0;
                for (String value : record) {
                    assertEquals(value, phoenixResultSet.getString(i + 1));
                    i++;
                }
            }
            assertFalse(phoenixResultSet.next());
        } finally {
            if (parser != null)
                parser.close();
            if (conn != null)
                conn.close();
        }
    }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.commons.csv.CSVParser

Chapter05

com.netflix.astyanax.util.CsvColumnReader

com.netflix.astyanax.util.CsvRecordReader

de.apaxo.bedcon.FacebookRecommender

name.abuchen.portfolio.datatransfer.CSVImporter

net.sf.uadetector.DeviceCategoryExamplesReader

net.sf.uadetector.OperatingSystemExamplesReader

net.sf.uadetector.OperatingSystemSampleReader

net.sf.uadetector.UserAgentExamplesReader

org.apache.any23.extractor.csv.CSVReaderBuilder

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.

Examples of org.apache.commons.csv.CSVParser

Parsing into memory

Notes

Related Classes of org.apache.commons.csv.CSVParser