Skip to content

Commit 1f2d53c

Browse files
totocaca123totocaca123
authored
Merge pull request #427 from totocaca123/manage_csv_delimiter
add_csv_delimiter detect
2 parents 35dffd0 + d19411e commit 1f2d53c

File tree

4 files changed

+87
-21
lines changed

4 files changed

+87
-21
lines changed

‎etc/Jenkinsfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ pipeline {
99
publishReport name: "Second JSON Report", displayType: "dual", provider: json(pattern: "etc/report-2.json")
1010
publishReport name: "XML Report", displayType: "dual", provider: xml(pattern: "etc/*.xml")
1111
publishReport name: "YAML Report", displayType: "dual", provider: yaml(pattern: "etc/*.yaml")
12+
1213
publishReport name: "CSV Report Test", displayType: "dual", provider: csv(id: "csv-one", pattern: "etc/report.csv")
14+
publishReport name: "CSV Report Test Tabs", displayType: "dual", provider: csv(id: "csv-one", pattern: "etc/report_tabulation.csv")
15+
publishReport name: "CSV Report Test SemiColon", displayType: "dual", provider: csv(id: "csv-one", pattern: "etc/report_semicolon.csv")
16+
1317
publishReport name: "CSV Report Test Robustness", displayType: "dual", provider: csv(id: "csv-one", pattern: "etc/report_robust.csv")
1418
}
1519
}

‎etc/report_semicolon.csv‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Level 1;Level 2;Level 3;incorrect;manually;accurate
2+
Aktie;Aktie 1;Aktie 1 1;13;3;59
3+
Aktie;Aktie 1;Aktie 1 2;31;48;9
4+
Aktie;Aktie 1;Aktie 1 3;34;8;51
5+
Derivat;Derivat 1;;1;2;3
6+
Derivat;Derivat 2;;5;9;1
7+
Not Found;;;1;3;9

‎etc/report_tabulation.csv‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Level 1 Level 2 Level 3 incorrect manually accurate
2+
Aktie Aktie 1 Aktie 1 1 13 3 59
3+
Aktie Aktie 1 Aktie 1 2 31 48 9
4+
Aktie Aktie 1 Aktie 1 3 34 8 51
5+
Derivat Derivat 1 1 2 3
6+
Derivat Derivat 2 5 9 1
7+
Not Found 1 3 9

‎src/main/java/io/jenkins/plugins/reporter/provider/Csv.java‎

Lines changed: 69 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,27 +17,31 @@
1717
import org.kohsuke.stapler.DataBoundConstructor;
1818

1919
import java.io.File;
20+
import java.io.BufferedReader;
2021
import java.io.IOException;
2122
import java.util.*;
23+
import java.io.FileInputStream;
24+
import java.io.InputStreamReader;
25+
import java.nio.charset.StandardCharsets;
2226

2327
public class Csv extends Provider {
24-
28+
2529
private static final long serialVersionUID = 9141170397250309265L;
2630

2731
private static final String ID = "csv";
28-
32+
2933
@DataBoundConstructor
3034
public Csv() {
3135
super();
3236
// empty constructor required for stapler
3337
}
34-
38+
3539
@Override
3640
public ReportParser createParser() {
3741
if (getActualId().equals(getDescriptor().getId())) {
3842
throw new IllegalArgumentException(Messages.Provider_Error());
3943
}
40-
44+
4145
return new CsvCustomParser(getActualId());
4246
}
4347

@@ -54,9 +58,9 @@ public Descriptor() {
5458
public static class CsvCustomParser extends ReportParser {
5559

5660
private static final long serialVersionUID = -8689695008930386640L;
57-
61+
5862
private final String id;
59-
63+
6064
private List<String> parserMessages;
6165

6266
public CsvCustomParser(String id) {
@@ -69,17 +73,54 @@ public String getId() {
6973
return id;
7074
}
7175

76+
77+
private char detectDelimiter(File file) throws IOException {
78+
// List of possible delimiters
79+
char[] delimiters = { ',', ';', '\t', '|' };
80+
int[] delimiterCounts = new int[delimiters.length];
81+
82+
// Read the lines of the file to detect the delimiter
83+
try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8))) {
84+
int linesToCheck = 5; // Number of lines to check
85+
int linesChecked = 0;
86+
87+
String line;
88+
while ((line = reader.readLine()) != null && linesChecked < linesToCheck) {
89+
for (int i = 0; i < delimiters.length; i++) {
90+
delimiterCounts[i] += StringUtils.countMatches(line, delimiters[i]);
91+
}
92+
linesChecked++;
93+
}
94+
}
95+
96+
// Return the most frequent delimiter
97+
int maxCount = 0;
98+
char detectedDelimiter = 0;
99+
for (int i = 0; i < delimiters.length; i++) {
100+
if (delimiterCounts[i] > maxCount) {
101+
maxCount = delimiterCounts[i];
102+
detectedDelimiter = delimiters[i];
103+
}
104+
}
105+
106+
return detectedDelimiter;
107+
}
108+
109+
72110
@Override
73111
public ReportDto parse(File file) throws IOException {
112+
// Get delimiter
113+
char delimiter = detectDelimiter(file);
74114

75115
final CsvMapper mapper = new CsvMapper();
76-
final CsvSchema schema = mapper.schemaFor(String[].class).withColumnSeparator(',');
116+
final CsvSchema schema = mapper.schemaFor(String[].class).withColumnSeparator(delimiter);
77117

78118
mapper.enable(CsvParser.Feature.WRAP_AS_ARRAY);
79119
mapper.enable(CsvParser.Feature.SKIP_EMPTY_LINES);
80120
mapper.enable(CsvParser.Feature.ALLOW_TRAILING_COMMA);
81121
mapper.enable(CsvParser.Feature.INSERT_NULLS_FOR_MISSING_COLUMNS);
82122
mapper.enable(CsvParser.Feature.TRIM_SPACES);
123+
83124
final MappingIterator<List<String>> it = mapper.readerForListOf(String.class)
84125
.with(schema)
85126
.readValues(file);
@@ -100,6 +141,7 @@ public ReportDto parse(File file) throws IOException {
100141
} else {
101142
parserMessages.add(String.format("skipped file - First line has %d elements", headerColumnCount + 1));
102143
}
144+
103145
/** Parse all data rows */
104146
for (int rowIdx = 0; rowIdx < rowCount; rowIdx++) {
105147
String parentId = "report";
@@ -116,20 +158,22 @@ public ReportDto parse(File file) throws IOException {
116158
for (int colIdx = rowSize - 1; colIdx > 1; colIdx--) {
117159
String value = row.get(colIdx);
118160

119-
if (NumberUtils.isCreatable(value) == true) {
161+
if (NumberUtils.isCreatable(value)) {
120162
colIdxValueStart = colIdx;
121163
} else {
122164
if (colIdxValueStart > 0) {
123-
parserMessages.add(String.format("Found data - fields number = %d - numeric fields = %d", colIdxValueStart, rowSize - colIdxValueStart));
165+
parserMessages
166+
.add(String.format("Found data - fields number = %d - numeric fields = %d",
167+
colIdxValueStart, rowSize - colIdxValueStart));
124168
}
125169
break;
126170
}
127171
}
128172
}
129173

130-
String valueId = "";
174+
String valueId = "";
131175
/** Parse line if first data line is OK and line has more element than header */
132-
if ((colIdxValueStart > 0) && (rowSize >= headerColumnCount)){
176+
if ((colIdxValueStart > 0) && (rowSize >= headerColumnCount)) {
133177
/** Check line and header size matching */
134178
for (int colIdx = 0; colIdx < headerColumnCount; colIdx++) {
135179
String id = header.get(colIdx);
@@ -141,19 +185,22 @@ public ReportDto parse(File file) throws IOException {
141185
if ((NumberUtils.isCreatable(value)) || (StringUtils.isBlank(value))) {
142186
/** Empty field found - message */
143187
if (colIdx == 0) {
144-
parserMessages.add(String.format("skipped line %d - First column item empty - col = %d ", rowIdx + 2, colIdx + 1));
145-
break ;
188+
parserMessages
189+
.add(String.format("skipped line %d - First column item empty - col = %d ",
190+
rowIdx + 2, colIdx + 1));
191+
break;
146192
} else {
147193
emptyFieldFound = true;
148194
/** Continue next column parsing */
149195
continue;
150196
}
151197
} else {
152198
/** Check if field values are present after empty cells */
153-
if (emptyFieldFound == true) {
154-
parserMessages.add(String.format("skipped line %d Empty field in col = %d ", rowIdx + 2, colIdx + 1));
155-
break;
156-
}
199+
if (emptyFieldFound) {
200+
parserMessages.add(String.format("skipped line %d Empty field in col = %d ",
201+
rowIdx + 2, colIdx + 1));
202+
break;
203+
}
157204
}
158205
valueId += value;
159206
Optional<Item> parent = report.findItem(parentId, report.getItems());
@@ -193,19 +240,20 @@ public ReportDto parse(File file) throws IOException {
193240
parserMessages.add(String.format("skipped line %d - First data row not found", rowIdx + 2));
194241
continue;
195242
} else {
196-
parserMessages.add(String.format("skipped line %d - line has fewer element than title", rowIdx + 2));
243+
parserMessages
244+
.add(String.format("skipped line %d - line has fewer element than title", rowIdx + 2));
197245
continue;
198246
}
199247
}
200248
/** If last item was created, it will be added to report */
201-
if (lastItemAdded == true) {
249+
if (lastItemAdded) {
202250
last.setResult(result);
203251
} else {
204252
parserMessages.add(String.format("ignored line %d - Same fields already exists", rowIdx + 2));
205253
}
206254
}
207-
//report.setParserLog(parserMessages);
255+
// report.setParserLog(parserMessages);
208256
return report;
209257
}
210258
}
211-
}
259+
}

0 commit comments

Comments
 (0)