Package au.com.bytecode.opencsv

Examples of au.com.bytecode.opencsv.CSVParser

167
168
169
170
171
172
173
174
175
176
177
   */
  private void loadItems() {
    itemCache = new HashMap<Integer, CSVDatastoreDAO.CSVItem>();
    try {
      CSVReader reader = new CSVReader(new InputStreamReader(
          CSVDatastoreDAO.class.getResourceAsStream("/db/items.csv")), 1/*skip header*/, new CSVParser('\t','"','\\'));
     
      String[] line = reader.readNext();
      while (line != null) {
        if (line.length > 2) {
          CSVItem n = new CSVItem();
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
            sep = "\\t";
        }
        sep = StringEscapeUtils.unescapeJava(sep);
        boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true);
       
        final CSVParser parser = new CSVParser(
            sep.toCharArray()[0],//HACK changing string to char - won't work for multi-char separators.
            CSVParser.DEFAULT_QUOTE_CHARACTER,
            (char) 0, // escape character
            CSVParser.DEFAULT_STRICT_QUOTES,
            CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE,
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    );   

    @Override
    public Object call(Properties bindings, Object[] args) {
        if (args.length >= 1 && args.length <= 2) {
            CSVParser parser = null;
           
            Object v = args[0];
            String s = v.toString();
           
            if (args.length > 1) {
                String sep = args[1].toString();
                parser = new CSVParser(
                    sep.charAt(0),
                    CSVParser.DEFAULT_QUOTE_CHARACTER,
                    CSVParser.DEFAULT_ESCAPE_CHARACTER,
                    CSVParser.DEFAULT_STRICT_QUOTES,
                    CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE,
                    false
                );
            }
           
            if (parser == null) {
                int tab = s.indexOf('\t');
                if (tab >= 0) {
                    parser = s_tabParser;
                } else {
                    parser = s_commaParser;
                }
            }
           
            try {
                return parser.parseLine(s);
            } catch (IOException e) {
                return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " error: " + e.getMessage());
            }
        }
        return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 1 or 2 strings");
View Full Code Here
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
        zis = getZipFile(zipBlobKey);

        ZipEntry ze;

        CSVParser p = new CSVParser();

        while ((ze = zis.getNextEntry()) != null) {

          final ByteArrayOutputStream baos = new ByteArrayOutputStream();

          final byte[] b = new byte[2048];

          int size = 0;
          while ((size = zis.read(b, 0, b.length)) != -1) {
            baos.write(b, 0, size);
          }

          String[] parts = ze.getName().split("/");
          String fileName = parts[parts.length - 1];

          if (fileName.contains("disks.csv")) {

            final String csv = new String(baos.toByteArray());

            List<?> headers = null;
            boolean h = true;

            for (String line : csv.split("\n")) {

              line = line.trim();

              // first row
              if (h) {
                headers = Arrays.asList(p.parseLine(line));
                h = false;
              }

              // data
              else {
                final String[] diskData = p.parseLine(line);

                final int length = diskData.length;
                if (length <= 1) {
                  break;
                }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
        _values = null;
    }

    private String[] getValuesInternal() {
        if (_values == null) {
            final CSVParser parser = _dataSet.getCsvParser();
            final String[] csvValues;
            try {
                csvValues = parser.parseLine(_line);
            } catch (IOException e) {
                throw new MetaModelException("Failed to parse CSV line no. " + _rowNumber + ": " + _line);
            }

            if (_failOnInconsistentRowLength) {
View Full Code Here
228
229
230
231
232
233
234
235
236
237
238
239
240
      return;
    }

    //step 2) parse the value

    CSVParser regexParser = new CSVParser('/', (char)0x0, '\\');

    String[] parsedRegex = regexParser.parseLine(valueStr);

    //DEBUG
    if (_DEBUG) System.out.println("pKV4: parsed regex = " + Arrays.toString(parsedRegex));
   
    //0 is "" or s
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    String line;
    List<DocumentPojo> partials = new LinkedList<DocumentPojo>();
    int docs = 0;
    _memUsage.memory = 0;
   
    CSVParser parser = null;
    Object[] indexToField = null;
    // *Automated* parser, else will just grab the line and let subsequent pipeline elements extract the fields
    if ((null != source.getFileConfig()) && (
         ((null != source.getFileConfig().XmlIgnoreValues) && (!source.getFileConfig().XmlIgnoreValues.isEmpty())) ||
         ((null != source.getFileConfig().XmlRootLevelValues) && (!source.getFileConfig().XmlRootLevelValues.isEmpty()))
      ))
    {
      if (null != source.getFileConfig().XmlAttributePrefix) {
        String chars = source.getFileConfig().XmlAttributePrefix;
        if (1 == chars.length()) {
          parser = new CSVParser(chars.charAt(0));
        }
        else if (2 == chars.length()) {
          parser = new CSVParser(chars.charAt(0), chars.charAt(1));
        }
        else if (chars.length() > 2) {
          parser = new CSVParser(chars.charAt(0), chars.charAt(1), chars.charAt(2));
        }
        if (chars.length() > 1) {
          _quoteChar = chars.charAt(1);
        }
      }
      if (null == parser) {
        parser = new CSVParser();
      }
      if ((null != source.getFileConfig().XmlRootLevelValues) && (source.getFileConfig().XmlRootLevelValues.size() > 0)) {
        indexToField = source.getFileConfig().XmlRootLevelValues.toArray();
      }
    }//TESTED
   
    boolean foundHeaderLine = (indexToField != null);
    while ((line = lineReader.readLine()) != null) {
      // Ignore header lines:
      if ((null != source.getFileConfig()) && (null != source.getFileConfig().XmlIgnoreValues)) {
        boolean bMatched = false;
        boolean firstIgnoreField = true; // (first ignore field in list can generate the headers)
        for (String ignore: source.getFileConfig().XmlIgnoreValues) {     
          boolean lineMatches = false;
          if (ignore.charAt(0) == _quoteChar) {
            if (line.charAt(0) == _quoteChar) {
              lineMatches = line.startsWith(ignore);
            }//TESTED (["a","b","c"] and XmlIgnoreFields: [ "\"a" ])             
            else {
              lineMatches = line.startsWith(ignore.substring(1));
            }//TESTED ([a,b,c] vs XmlIgnoreFields: [ "a" ] and [ "\"a" ])             
          }
          else {
            lineMatches = line.startsWith(ignore);
          }//TESTED
         
          if (lineMatches) {
            if (!foundHeaderLine && firstIgnoreField && (null != parser)) {
              if (ignore.charAt(0) != _quoteChar) { // if using quotes then don't pull the char
                line = line.substring(ignore.length());
              }//TESTED (["a","b","c"] and [a,b,c] vs XmlIgnoreFields: [ "a" ] and [ "\"a" ])             
              String[] fields = parser.parseLine(line);
              // Now override the manual fields:
              indexToField = Arrays.asList(fields).toArray();
             
              if ((indexToField.length > 1) || (0 != ((String)indexToField[0]).length())) {
                foundHeaderLine = true;
              }//TESTED
            }//TESTED
            bMatched = true;
          }
          firstIgnoreField = false;
        }
        if (bMatched) continue;
      }//TESTED
     
      DocumentPojo newDoc = new DocumentPojo();
      String primaryKey = null;
      if (null != parser) {
        JsonObject json = new JsonObject();
        try {
          String[] records = parser.parseLine(line);
          for (int i = 0; i < records.length; ++i) {
            String record = records[i];
            if ((record.length() > 0) && (i < indexToField.length)) {
              String fieldName = (String) indexToField[i];
              if ((null != fieldName) && (fieldName.length() > 0)) {
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
      ))
    {
      if (null != fileConfig.XmlAttributePrefix) {
        String chars = fileConfig.XmlAttributePrefix;
        if (1 == chars.length()) {
          parser = new CSVParser(chars.charAt(0));
        }
        else if (2 == chars.length()) {
          parser = new CSVParser(chars.charAt(0), chars.charAt(1));
        }
        else if (chars.length() > 2) {
          parser = new CSVParser(chars.charAt(0), chars.charAt(1), chars.charAt(2));
        }
        if (chars.length() > 1) {
          _quoteChar = chars.charAt(1);
        }
      }
      if (null == parser) {
        parser = new CSVParser();
      }
      if ((null != fileConfig.XmlRootLevelValues) && (fileConfig.XmlRootLevelValues.size() > 0)) {
        indexToField = fileConfig.XmlRootLevelValues.toArray();
      }
    }//TESTED
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
  public static boolean isCSV(FileSystem fs, Path p) {
    String fname = p.getName();   
    if (fname.endsWith(".csv")) {
      return true;
    }
    CSVParser parser = new CSVParser();
    try {
      BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(p)));
      try {
        int lineCount = 0;
        List<Integer> observedEltCounts = new ArrayList<Integer>();
        int totalEltCount = 0;
        int minEltCount = Integer.MAX_VALUE;
        int maxEltCount = -1;

        String line = null;
        while (lineCount < MAX_LINES && ((line = in.readLine()) != null)) {
          String parts[] = parser.parseLine(line);
          int numElts = parts.length;
          minEltCount = Math.min(minEltCount, numElts);
          maxEltCount = Math.max(maxEltCount, numElts);
          totalEltCount += numElts;
          observedEltCounts.add(numElts);
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
    //
    int numColumns = 0;
    String firstLine = null;
    List<String> firstRow = new ArrayList<String>();
    List<List<Schema.Type>> allEltTypes = new ArrayList<List<Schema.Type>>();
    CSVParser parser = new CSVParser();   
    BufferedReader in = new BufferedReader(new InputStreamReader(dd.getRawBytes()));
    try {
      int lineno = 0;
      String s = null;
      while ((s = in.readLine()) != null) {
        List<Schema.Type> schemaTypes = new ArrayList<Schema.Type>();
        String parts[] = parser.parseLine(s);

        for (int i = 0; i < parts.length; i++) {
          String elt = parts[i];
          if (elt.startsWith(",")) {
            elt = elt.substring(1);
View Full Code Here
TOP

Related Classes of au.com.bytecode.opencsv.CSVParser

  • com.antonytrupe.server.battledisks.API
  • com.cloudera.recordbreaker.analyzer.CSVDataDescriptor
  • com.cloudera.recordbreaker.analyzer.CSVRowParser
  • com.cloudera.recordbreaker.analyzer.CSVSchemaDescriptor
  • com.google.refine.expr.functions.strings.SmartSplit
  • com.google.refine.importers.SeparatorBasedImporter
  • com.ikanow.infinit.e.data_model.custom.InfiniteFileInputCsvParser
  • com.ikanow.infinit.e.harvest.enrichment.legacy.RegexEntityExtractor
  • com.ikanow.infinit.e.harvest.extraction.document.file.CsvToMetadataParser
  • com.l2client.dao.csv.CSVDatastoreDAO
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.