CSV Processing

1"{\"a\":\"#\",\"b\":\"b value\"}"
2Error: Data between close double quote (\") and field separator."; Reason: "invalid"},Invalid {Location: ""; Message: "Error while reading data, error message: CSV processing encountered too many errors, giving up. Rows: 0; errors: 1; max bad: 0; error percent: 0"; Reason: "invalid"}
3"{""a"":""#"",""b"":""b value""}"
4

1COL_A,COL_B,COL_F,COL_H,COL_M,COL_T,COL_Z
21,2,3,4,5,6,7
38,9,10,11,12,13,14
4...
5def process_csv(file):
6    with open(file, 'r') as f:
7        headers = f.readline().split(',')
8        has_C = "COL_C" in headers
9        has_D = "COL_D" in headers
10        has_M = "COL_M" in headers
11        has_N = "COL_N" in headers
12        
13        for line in f:
14            elements = line.split(',')
15            if has_C:
16                ...  # Special processing related to COL_C
17            if has_D:
18                ...  # Special processing related to COL_D
19            if has_M:
20                ...  # Special processing related to COL_M
21            if has_N:
22                ...  # Special processing related to COL_N
23            ...  # General processing, common to all iterations
24def new_process_csv(file):
25    with open(file, 'r') as f:
26        headers = f.readline().split(',')
27        code = \
28f'''
29for line in f:
30    elements = line.split(',')
31    {... if "COL_C" in headers else ''}
32    {... if "COL_D" in headers else ''}
33    {... if "COL_M" in headers else ''}
34    {... if "COL_N" in headers else ''}
35    ...  # General processing   
36'''
37        exec(code)
38        has_C = "COL_C" in headers
39        has_D = "COL_D" in headers
40        has_M = "COL_M" in headers
41        has_N = "COL_N" in headers
42

1COL_A,COL_B,COL_F,COL_H,COL_M,COL_T,COL_Z
21,2,3,4,5,6,7
38,9,10,11,12,13,14
4...
5def process_csv(file):
6    with open(file, 'r') as f:
7        headers = f.readline().split(',')
8        has_C = "COL_C" in headers
9        has_D = "COL_D" in headers
10        has_M = "COL_M" in headers
11        has_N = "COL_N" in headers
12        
13        for line in f:
14            elements = line.split(',')
15            if has_C:
16                ...  # Special processing related to COL_C
17            if has_D:
18                ...  # Special processing related to COL_D
19            if has_M:
20                ...  # Special processing related to COL_M
21            if has_N:
22                ...  # Special processing related to COL_N
23            ...  # General processing, common to all iterations
24def new_process_csv(file):
25    with open(file, 'r') as f:
26        headers = f.readline().split(',')
27        code = \
28f'''
29for line in f:
30    elements = line.split(',')
31    {... if "COL_C" in headers else ''}
32    {... if "COL_D" in headers else ''}
33    {... if "COL_M" in headers else ''}
34    {... if "COL_N" in headers else ''}
35    ...  # General processing   
36'''
37        exec(code)
38        has_C = "COL_C" in headers
39        has_D = "COL_D" in headers
40        has_M = "COL_M" in headers
41        has_N = "COL_N" in headers
42        headers = f.readline().split(',')
43        headers_set = set(headers)
44        
45        for line in f:
46            elements = line.split(',')
47            if "COL_C" in headers_set:
48                ...  # Special processing related to COL_C
49            if "COL_D" in headers_set:
50                ...  # Special processing related to COL_D
51            if "COL_M" in headers_set:
52                ...  # Special processing related to COL_M
53            if "COL_N" in headers_set:
54                ...  # Special processing related to COL_N
55

1runtime: program exceeds 10000-thread limit
2// main func (start threads)
3
4for i := range filelist {
5  channels = append(channels, make(chan Test))
6  go Parse(files[i], channels[len(channels)-1])
7}
8
9// Parse func (individual threads)
10
11func Parse(fileName string, c chan Test) {
12defer close(c)
13
14doc := etree.NewDocument()
15if err := doc.ReadFromFile(fileName); err != nil {
16    return
17}
18
19root := doc.SelectElement("trc:TestResultsCollection")
20
21for _, test := range root.FindElements("//trc:TestResults/tr:ResultSet/tr:TestGroup/tr:Test") {
22    var outcome Test
23    outcome.StepType = test.FindElement("./tr:Extension/ts:TSStepProperties/ts:StepType").Text()
24    outcome.Result = test.FindElement("./tr:Outcome").Attr[0].Value
25    for _, attr := range test.Attr {
26        if attr.Key == "name" {
27            outcome.Name = attr.Value
28        }
29    }
30
31    for _, attr := range test.FindElement("./tr:TestResult/tr:TestData/c:Datum").Attr {
32        if attr.Key == "value" {
33            outcome.Value = attr.Value
34        }
35    }
36
37    c <- outcome
38}
39
40}
41
42// main (process results when threads return)
43
44for c := 0; c < len(channels); c++ {
45    for i := range channels[c] {
46        // csv processing with i
47    }
48}
49jobs := make(chan string, numJobs)
50results := make(chan []Test, numJobs)
51
52for w := 0; w < numWorkers; w++ {
53    go Worker(w, jobs, results)
54    wg.Add(1)
55}
56
57// give workers jobs
58
59for _, i := range files {
60    if filepath.Ext(i) == ".xml" {
61        jobs <- ("Path to files" + i)
62    }
63}
64
65close(jobs)
66wg.Wait()
67
68//result processing <- results
69

1std::vector<std::string> string_to_vector(const std::string& s, const char delimiter, const char escape) {
2  std::stringstream sstr{s};
3  std::vector<std::string> result;
4  while (sstr.good()) {
5    std::string substr;
6    getline(sstr, substr, delimiter);
7    while (substr.back() == escape) {
8      std::string tmp;
9      getline(sstr, tmp, delimiter);
10      substr += "," + tmp;
11    }
12    result.emplace_back(substr);
13  }
14  return result;
15}
16#include <iostream>
17#include <vector>
18#include <string>
19
20std::vector<char*> string_to_vector(std::string& s, 
21                                    const char delimiter, const char escape) 
22{
23  size_t prev(0), pos(0), from(0);
24  std::vector<char*> v;
25  while ((pos = s.find(delimiter, from)) != s.npos)
26  {
27    if (pos == 0 || s[pos - 1] != escape)
28    {
29      s[pos] = 0;
30      v.push_back(&s[prev]);
31      prev = pos + 1;
32    }
33    from = pos + 1;
34  }
35  v.push_back(&s[prev]);
36  return v;
37}
38
39int main() {
40  std::string test("this,is,a\\,test");
41  std::vector<char*> v = string_to_vector(test, ',', '\\');
42
43  for (auto& s : v)
44    std::cout << s << " ";
45}
46

1
2void main() {
3
4  String delimiter = ",";
5  String rawRow = ',,"data",moredata,"Data"';
6RegExp exp = new RegExp(r'(?:'+ delimiter + r'"|^")(^,|""|[\w\W]*?)(?="'+ delimiter + r'|"$)|(?:'+ delimiter + '(?!")|^(?!"))([^'+ delimiter + r']*?)(?=$|'+ delimiter + r')');
7
8
9Iterable<Match> matches = exp.allMatches(rawRow.replaceAll("\n","").replaceAll("\r","").trim());
10List<String> row = new List();
11matches.forEach((Match m) {
12  //This checks to see which match group it found the item in.
13  String cellValue;
14  if (m.group(2) != null) {
15    //Data found without speech marks
16    cellValue = m.group(2);
17  } else if (m.group(1) != null) {
18    //Data found with speech marks (so it removes escaped quotes)
19    cellValue = m.group(1).replaceAll('""', '"');
20  }  else {
21    //Anything left
22    cellValue = m.group(0).replaceAll('""', '"');
23  }
24  row.add(cellValue);
25});
26  print(row.toString());
27
28}
29(,"|^")
30(""|[\w\W]*?)
31(?=",|"$)
32|
33(,(?!")|^(?!"))
34([^,]*?|)
35(?=$|,)
36