Support non-utf8 encoding for zip files

go.mod

@@ -58,6 +58,7 @@ require (
 	github.com/vearutop/statigz v1.1.6
 	github.com/vektah/dataloaden v0.3.0
 	github.com/vektah/gqlparser/v2 v2.4.1
+	github.com/xWTF/chardet v0.0.0-20230208095535-c780f2ac244e
 	gopkg.in/guregu/null.v4 v4.0.0
 )
 

go.sum

@@ -760,6 +760,8 @@ github.com/vektah/gqlparser/v2 v2.4.1/go.mod h1:flJWIR04IMQPGz+BXLrORkrARBxv/rty
 github.com/vektra/mockery/v2 v2.10.0 h1:MiiQWxwdq7/ET6dCXLaJzSGEN17k758H7JHS9kOdiks=
 github.com/vektra/mockery/v2 v2.10.0/go.mod h1:m/WO2UzWzqgVX3nvqpRQq70I4Z7jbSCRhdmkgtp+Ab4=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/xWTF/chardet v0.0.0-20230208095535-c780f2ac244e h1:GruPsb+44XvYAzuAgJW1d1WHqmcI73L2XSjsbx/eJZw=
+github.com/xWTF/chardet v0.0.0-20230208095535-c780f2ac244e/go.mod h1:wA8kQ8WFipMciY9WcWzqQgZordm/P7l8IZdvx1crwmc=
 github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=

pkg/file/zip.go

@@ -2,11 +2,18 @@ package file
 
 import (
 	"archive/zip"
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
 	"io/fs"
 	"path/filepath"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/xWTF/chardet"
+
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
 )
 
 var (
@@ -40,6 +47,42 @@ func newZipFS(fs FS, path string, info fs.FileInfo) (*ZipFS, error) {
 		return nil, err
 	}
 
+	// Concat all Name and Comment for better detection result
+	var buffer bytes.Buffer
+	for _, f := range zipReader.File {
+		buffer.WriteString(f.Name)
+		buffer.WriteString(f.Comment)
+	}
+	buffer.WriteString(zipReader.Comment)
+
+	// Detect encoding
+	d, err := chardet.NewTextDetector().DetectBest(buffer.Bytes())
+	if err != nil {
+		reader.Close()
+		return nil, fmt.Errorf("unable to detect decoding: %w", err)
+	}
+
+	// If the charset is not UTF8, decode'em
+	if d.Charset != "UTF-8" {
+		logger.Debugf("Detected non-utf8 zip charset %s (%s): %s", d.Charset, d.Language, path)
+
+		e, _ := charset.Lookup(d.Charset)
+		if e == nil {
+			reader.Close()
+			return nil, fmt.Errorf("failed to lookup charset %s, language %s", d.Charset, d.Language)
+		}
+
+		decoder := e.NewDecoder()
+		for _, f := range zipReader.File {
+			f.Name, _, err = transform.String(decoder, f.Name)
+			if err != nil {
+				reader.Close()
+				return nil, fmt.Errorf("failed to decode %v: %w", []byte(f.Name), err)
+			}
+			// Comments are not decoded cuz stash doesn't use that
+		}
+	}
+
 	return &ZipFS{
 		Reader:        zipReader,
 		zipFileCloser: reader,