Merge pull request #3168 from alibaba/bugfix

Bugfix

Author: zhuangjiaju

README.md

@@ -31,7 +31,7 @@ easyexcel重写了poi对07版Excel的解析，一个3M的excel用POI sax解析
 <dependency>
     <groupId>com.alibaba</groupId>
     <artifactId>easyexcel</artifactId>
-    <version>3.2.1</version>
+    <version>3.3.0</version>
 </dependency>
 ```
 

README_EN.md

@@ -122,7 +122,7 @@ DEMO：[https://github.com/alibaba/easyexcel/blob/master/src/test/java/com/aliba
      *  1. Create an entity object, refer to{@link DownloadData}. 
      *     Each property of the entity object corresponds to a specific field of Excel
      *  2. Specify the returned properties
-     *  3. Invoke wirte function, then the OutputStream is automatically closed when it ends.
+     *  3. Invoke write function, then the OutputStream is automatically closed when it ends.
      * </p>
      */
     @GetMapping("download")

easyexcel-core/pom.xml

@@ -40,5 +40,9 @@
             <groupId>org.ehcache</groupId>
             <artifactId>ehcache</artifactId>
         </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
     </dependencies>
 </project>

easyexcel-core/src/main/java/com/alibaba/excel/analysis/ExcelAnalyserImpl.java

@@ -19,6 +19,7 @@
 import com.alibaba.excel.read.metadata.holder.xls.XlsReadWorkbookHolder;
 import com.alibaba.excel.read.metadata.holder.xlsx.XlsxReadWorkbookHolder;
 import com.alibaba.excel.support.ExcelTypeEnum;
+import com.alibaba.excel.util.ClassUtils;
 import com.alibaba.excel.util.DateUtils;
 import com.alibaba.excel.util.FileUtils;
 import com.alibaba.excel.util.NumberDataFormatterUtils;
@@ -212,6 +213,7 @@ public void finish() {
     private void removeThreadLocalCache() {
         NumberDataFormatterUtils.removeThreadLocalCache();
         DateUtils.removeThreadLocalCache();
+        ClassUtils.removeThreadLocalCache();
     }
 
     private void clearEncrypt03() {

easyexcel-core/src/main/java/com/alibaba/excel/analysis/csv/CsvExcelReadExecutor.java

@@ -1,6 +1,7 @@
 package com.alibaba.excel.analysis.csv;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.file.Files;
 import java.util.ArrayList;
@@ -11,6 +12,7 @@
 
 import com.alibaba.excel.analysis.ExcelReadExecutor;
 import com.alibaba.excel.context.csv.CsvReadContext;
+import com.alibaba.excel.enums.ByteOrderMarkEnum;
 import com.alibaba.excel.enums.CellDataTypeEnum;
 import com.alibaba.excel.enums.RowTypeEnum;
 import com.alibaba.excel.exception.ExcelAnalysisException;
@@ -27,6 +29,7 @@
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.io.input.BOMInputStream;
 
 /**
  * read executor
@@ -82,17 +85,26 @@ public void execute() {
     private CSVParser csvParser() throws IOException {
         CsvReadWorkbookHolder csvReadWorkbookHolder = csvReadContext.csvReadWorkbookHolder();
         CSVFormat csvFormat = csvReadWorkbookHolder.getCsvFormat();
-
+        ByteOrderMarkEnum byteOrderMark = ByteOrderMarkEnum.valueOfByCharsetName(
+            csvReadContext.csvReadWorkbookHolder().getCharset().name());
         if (csvReadWorkbookHolder.getMandatoryUseInputStream()) {
-            return csvFormat.parse(
-                new InputStreamReader(csvReadWorkbookHolder.getInputStream(), csvReadWorkbookHolder.getCharset()));
+            return buildCsvParser(csvFormat, csvReadWorkbookHolder.getInputStream(), byteOrderMark);
         }
         if (csvReadWorkbookHolder.getFile() != null) {
-            return csvFormat.parse(new InputStreamReader(Files.newInputStream(csvReadWorkbookHolder.getFile().toPath()),
-                csvReadWorkbookHolder.getCharset()));
+            return buildCsvParser(csvFormat, Files.newInputStream(csvReadWorkbookHolder.getFile().toPath()),
+                byteOrderMark);
+        }
+        return buildCsvParser(csvFormat, csvReadWorkbookHolder.getInputStream(), byteOrderMark);
+    }
+
+    private CSVParser buildCsvParser(CSVFormat csvFormat, InputStream inputStream, ByteOrderMarkEnum byteOrderMark)
+        throws IOException {
+        if (byteOrderMark == null) {
+            return csvFormat.parse(
+                new InputStreamReader(inputStream, csvReadContext.csvReadWorkbookHolder().getCharset()));
         }
-        return csvFormat.parse(
-            new InputStreamReader(csvReadWorkbookHolder.getInputStream(), csvReadWorkbookHolder.getCharset()));
+        return csvFormat.parse(new InputStreamReader(new BOMInputStream(inputStream, byteOrderMark.getByteOrderMark()),
+            csvReadContext.csvReadWorkbookHolder().getCharset()));
     }
 
     private void dealRecord(CSVRecord record, int rowIndex) {

easyexcel-core/src/main/java/com/alibaba/excel/analysis/v07/handlers/sax/SharedStringsTableHandler.java

@@ -1,18 +1,40 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
 package com.alibaba.excel.analysis.v07.handlers.sax;
 
-import org.xml.sax.Attributes;
-import org.xml.sax.helpers.DefaultHandler;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import com.alibaba.excel.cache.ReadCache;
 import com.alibaba.excel.constant.ExcelXmlConstants;
 
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+
 /**
  * Sax read sharedStringsTable.xml
  *
  * @author Jiaju Zhuang
  */
 public class SharedStringsTableHandler extends DefaultHandler {
 
+    private static final Pattern UTF_PATTTERN = Pattern.compile("_x([0-9A-Fa-f]{4})_");
+
     /**
      * The final piece of data
      */
@@ -86,7 +108,7 @@ public void endElement(String uri, String localName, String name) {
                 if (currentData == null) {
                     readCache.put(null);
                 } else {
-                    readCache.put(currentData.toString());
+                    readCache.put(utfDecode(currentData.toString()));
                 }
                 break;
             case ExcelXmlConstants.SHAREDSTRINGS_RPH_TAG:
@@ -109,4 +131,51 @@ public void characters(char[] ch, int start, int length) {
         }
         currentElementData.append(ch, start, length);
     }
+
+    /**
+     * from poi XSSFRichTextString
+     *
+     * @param value the string to decode
+     * @return the decoded string or null if the input string is null
+     * <p>
+     * For all characters which cannot be represented in XML as defined by the XML 1.0 specification,
+     * the characters are escaped using the Unicode numerical character representation escape character
+     * format _xHHHH_, where H represents a hexadecimal character in the character's value.
+     * <p>
+     * Example: The Unicode character 0D is invalid in an XML 1.0 document,
+     * so it shall be escaped as <code>_x000D_</code>.
+     * </p>
+     * See section 3.18.9 in the OOXML spec.
+     * @see org.apache.poi.xssf.usermodel.XSSFRichTextString#utfDecode(String)
+     */
+    static String utfDecode(String value) {
+        if (value == null || !value.contains("_x")) {
+            return value;
+        }
+
+        StringBuilder buf = new StringBuilder();
+        Matcher m = UTF_PATTTERN.matcher(value);
+        int idx = 0;
+        while (m.find()) {
+            int pos = m.start();
+            if (pos > idx) {
+                buf.append(value, idx, pos);
+            }
+
+            String code = m.group(1);
+            int icode = Integer.decode("0x" + code);
+            buf.append((char)icode);
+
+            idx = m.end();
+        }
+
+        // small optimization: don't go via StringBuilder if not necessary,
+        // the encodings are very rare, so we should almost always go via this shortcut.
+        if (idx == 0) {
+            return value;
+        }
+
+        buf.append(value.substring(idx));
+        return buf.toString();
+    }
 }

easyexcel-core/src/main/java/com/alibaba/excel/context/WriteContextImpl.java

@@ -265,7 +265,7 @@ private void addOneRowOfHeadDataToExcel(Row row, Integer rowIndex, Map<Integer,
             Head head = entry.getValue();
             int columnIndex = entry.getKey();
             ExcelContentProperty excelContentProperty = ClassUtils.declaredExcelContentProperty(null,
-                currentWriteHolder.excelWriteHeadProperty().getHeadClazz(), head.getFieldName());
+                currentWriteHolder.excelWriteHeadProperty().getHeadClazz(), head.getFieldName(), currentWriteHolder);
 
             CellWriteHandlerContext cellWriteHandlerContext = WriteHandlerUtils.createCellWriteHandlerContext(this, row,
                 rowIndex, head, columnIndex, relativeRowIndex, Boolean.TRUE, excelContentProperty);
@@ -427,6 +427,7 @@ public void finish(boolean onException) {
     private void removeThreadLocalCache() {
         NumberDataFormatterUtils.removeThreadLocalCache();
         DateUtils.removeThreadLocalCache();
+        ClassUtils.removeThreadLocalCache();
     }
 
     @Override

easyexcel-core/src/main/java/com/alibaba/excel/converters/DefaultConverterLoader.java

@@ -115,8 +115,6 @@ private static void initAllConverter() {
         putAllConverter(new StringNumberConverter());
         putAllConverter(new StringStringConverter());
         putAllConverter(new StringErrorConverter());
-
-        putAllConverter(new BigIntegerStringConverter());
     }
 
     private static void initDefaultWriteConverter() {
@@ -154,7 +152,6 @@ private static void initDefaultWriteConverter() {
         putWriteStringConverter(new LongStringConverter());
         putWriteStringConverter(new ShortStringConverter());
         putWriteStringConverter(new StringStringConverter());
-        putWriteStringConverter(new BigIntegerStringConverter());
     }
 
     /**

easyexcel-core/src/main/java/com/alibaba/excel/enums/ByteOrderMarkEnum.java

@@ -0,0 +1,66 @@
+package com.alibaba.excel.enums;
+
+import java.nio.charset.Charset;
+import java.util.Map;
+
+import com.alibaba.excel.util.MapUtils;
+
+import lombok.Getter;
+import org.apache.commons.io.ByteOrderMark;
+
+/**
+ * byte order mark
+ *
+ * @author Jiaju Zhuang
+ */
+@Getter
+public enum ByteOrderMarkEnum {
+
+    /**
+     * UTF_8
+     */
+    UTF_8(ByteOrderMark.UTF_8),
+    /**
+     * UTF_16BE
+     */
+    UTF_16BE(ByteOrderMark.UTF_16BE),
+    /**
+     * UTF_16LE
+     */
+    UTF_16LE(ByteOrderMark.UTF_16LE),
+    /**
+     * UTF_32BE
+     */
+    UTF_32BE(ByteOrderMark.UTF_32BE),
+    /**
+     * UTF_32LE
+     */
+    UTF_32LE(ByteOrderMark.UTF_32LE),
+
+    ;
+
+    final ByteOrderMark byteOrderMark;
+    final String stringPrefix;
+
+    ByteOrderMarkEnum(ByteOrderMark byteOrderMark) {
+        this.byteOrderMark = byteOrderMark;
+        Charset charset = Charset.forName(byteOrderMark.getCharsetName());
+        this.stringPrefix = new String(byteOrderMark.getBytes(), charset);
+    }
+
+    /**
+     * store character aliases corresponding to `ByteOrderMark` prefix
+     */
+    private static final Map<String, ByteOrderMarkEnum> CHARSET_BYTE_ORDER_MARK_MAP = MapUtils.newHashMap();
+
+    static {
+        for (ByteOrderMarkEnum value : ByteOrderMarkEnum.values()) {
+            CHARSET_BYTE_ORDER_MARK_MAP.put(value.getByteOrderMark().getCharsetName(), value);
+        }
+    }
+
+    public static ByteOrderMarkEnum valueOfByCharsetName(String charsetName) {
+        return CHARSET_BYTE_ORDER_MARK_MAP.get(charsetName);
+    }
+
+}

easyexcel-core/src/main/java/com/alibaba/excel/enums/CacheLocationEnum.java

@@ -0,0 +1,23 @@
+package com.alibaba.excel.enums;
+
+/**
+ * cache locaciton
+ *
+ * @author Jiaju Zhuang
+ **/
+public enum CacheLocationEnum {
+    /**
+     * The cache will be stored in {@code ThreadLocal}, and will be cleared when the excel read and write is completed.
+     */
+    THREAD_LOCAL,
+
+    /**
+     * The cache will not be cleared unless the app is stopped.
+     */
+    MEMORY,
+
+    /**
+     * No caching.It may lose some of performance.
+     */
+    NONE;
+}