Skip to content

Commit 26225d7

Browse files
vitaminmoovitaminmoo
authored
Google Cloud Transparent GZIP Compression Support (#104)
* Initial commit of PoC of transparent snappy compression * Fix tests * fix up require usage * Fix up temp dir usage * Temp dirs but right * Delete custom buffered test writer thing * make csbufio reads peek to id snappy * Remove junk * Prep for using lzma * More * Convert to gzip * Fix google tests, handle errors, read compressed * Start removing local compression support * Remove enablecompression during testing * Remove more * Remove pointless redundancy * Make Content-Encoding available externally * Re-add accidentally removed test * Destandardize Content-Encoding to content_encoding * Add missing reference to enableCompression * Add missed write path * Set mime correctly * Handle additional codepath * Work around an interesting google library behavior * Do the other codepath too * Fix comment
1 parent a437124 commit 26225d7

File tree

21 files changed

+593
-463
lines changed

21 files changed

+593
-463
lines changed

‎awss3/store_test.go‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ package awss3_test
22

33
import (
44
"os"
5+
"path/filepath"
56
"testing"
67

78
"github.com/araddon/gou"
8-
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
910

1011
"github.com/lytics/cloudstorage"
1112
"github.com/lytics/cloudstorage/awss3"
@@ -28,6 +29,9 @@ func TestS3(t *testing.T) {
2829
t.Skip()
2930
return
3031
}
32+
33+
tmpDir := t.TempDir()
34+
3135
conf := &cloudstorage.Config{
3236
Type: awss3.StoreType,
3337
Settings: gou.JsonHelper{
@@ -36,53 +40,55 @@ func TestS3(t *testing.T) {
3640
}
3741
// Should error with empty config
3842
_, err := cloudstorage.NewStore(conf)
39-
assert.NotEqual(t, nil, err)
43+
require.Error(t, err)
4044

4145
conf.AuthMethod = awss3.AuthAccessKey
4246
conf.Settings[awss3.ConfKeyAccessKey] = ""
4347
conf.Settings[awss3.ConfKeyAccessSecret] = os.Getenv("AWS_SECRET_KEY")
4448
conf.Bucket = os.Getenv("AWS_BUCKET")
45-
conf.TmpDir = "/tmp/localcache/aws"
49+
conf.TmpDir = filepath.Join(tmpDir, "localcache", "aws")
4650
_, err = cloudstorage.NewStore(conf)
47-
assert.NotEqual(t, nil, err)
51+
require.Error(t, err)
4852

4953
conf.Settings[awss3.ConfKeyAccessSecret] = ""
5054
_, err = cloudstorage.NewStore(conf)
51-
assert.NotEqual(t, nil, err)
55+
require.Error(t, err)
5256

5357
// conf.Settings[awss3.ConfKeyAccessKey] = "bad"
5458
// conf.Settings[awss3.ConfKeyAccessSecret] = "bad"
5559
// _, err = cloudstorage.NewStore(conf)
56-
// assert.NotEqual(t, nil, err)
60+
// require. NotEqual(t, nil, err)
5761

5862
conf.BaseUrl = "s3.custom.endpoint.com"
5963
conf.Settings[awss3.ConfKeyAccessKey] = os.Getenv("AWS_ACCESS_KEY")
6064
conf.Settings[awss3.ConfKeyAccessSecret] = os.Getenv("AWS_SECRET_KEY")
61-
client, sess, err := awss3.NewClient(conf)
62-
assert.Equal(t, nil, err)
63-
assert.NotEqual(t, nil, client)
65+
client, _, err := awss3.NewClient(conf)
66+
require.NoError(t, err)
67+
require.NotNil(t, client)
6468

6569
conf.Settings[awss3.ConfKeyDisableSSL] = true
66-
client, sess, err = awss3.NewClient(conf)
67-
assert.Equal(t, nil, err)
68-
assert.NotEqual(t, nil, client)
70+
client, sess, err := awss3.NewClient(conf)
71+
require.NoError(t, err)
72+
require.NotNil(t, client)
6973

7074
conf.TmpDir = ""
7175
_, err = awss3.NewStore(client, sess, conf)
72-
assert.NotEqual(t, nil, err)
76+
require.Error(t, err)
7377

7478
// Trying to find dir they don't have access to?
7579
conf.TmpDir = "/home/fake"
7680
_, err = cloudstorage.NewStore(conf)
77-
assert.NotEqual(t, nil, err)
81+
require.Error(t, err)
7882
}
7983

8084
func TestAll(t *testing.T) {
85+
tmpDir := t.TempDir()
86+
8187
config := &cloudstorage.Config{
8288
Type: awss3.StoreType,
8389
AuthMethod: awss3.AuthAccessKey,
8490
Bucket: os.Getenv("AWS_BUCKET"),
85-
TmpDir: "/tmp/localcache/aws",
91+
TmpDir: filepath.Join(tmpDir, "localcache", "aws"),
8692
Settings: make(gou.JsonHelper),
8793
Region: "us-east-1",
8894
}
@@ -100,8 +106,6 @@ func TestAll(t *testing.T) {
100106
t.Skip()
101107
return
102108
}
103-
if store == nil {
104-
t.Fatalf("No store???")
105-
}
109+
require.NotNil(t, store, "no store?")
106110
testutils.RunTests(t, store, config)
107111
}

‎azure/README.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,19 @@ export AZURE_BUCKET="cloudstorageunittests"
4444
*/
4545

4646
func main() {
47+
tmpDir, err := os.MkdirTemp("/tmp", "azure_example")
48+
if err != nil {
49+
fmt.Println("Could not create temp dir", err)
50+
os.Exit(1)
51+
}
52+
defer os.RemoveAll(tmpDir)
53+
4754
conf := &cloudstorage.Config{
4855
Type: azure.StoreType,
4956
AuthMethod: azure.AuthKey,
5057
Bucket: os.Getenv("AZURE_BUCKET"),
5158
Project: os.Getenv("AZURE_PROJECT"),
52-
TmpDir: "/tmp/localcache/azure",
59+
TmpDir: filepath.Join(tempDir, "localcache", "azure"),
5360
Settings: make(gou.JsonHelper),
5461
}
5562

‎azure/example/main.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"path/filepath"
78

89
"github.com/araddon/gou"
910
"google.golang.org/api/iterator"
@@ -23,12 +24,19 @@ export AZURE_BUCKET="cloudstorageunittests"
2324
*/
2425

2526
func main() {
27+
tmpDir, err := os.MkdirTemp("/tmp", "azure_example")
28+
if err != nil {
29+
fmt.Println("Could not create temp dir", err)
30+
os.Exit(1)
31+
}
32+
defer os.RemoveAll(tmpDir)
33+
2634
conf := &cloudstorage.Config{
2735
Type: azure.StoreType,
2836
AuthMethod: azure.AuthKey,
2937
Bucket: os.Getenv("AZURE_BUCKET"),
3038
Project: os.Getenv("AZURE_PROJECT"),
31-
TmpDir: "/tmp/localcache/azure",
39+
TmpDir: filepath.Join(tmpDir, "localcache", "azure"),
3240
Settings: make(gou.JsonHelper),
3341
}
3442

‎azure/store.go‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -547,20 +547,20 @@ func newObject(f *FS, o *az.Blob) *object {
547547
}
548548

549549
/*
550-
func newObjectFromHead(f *FS, name string, o *s3.HeadObjectOutput) *object {
551-
obj := &object{
552-
fs: f,
553-
name: name,
554-
bucket: f.bucket,
555-
cachepath: cloudstorage.CachePathObj(f.cachepath, name, f.ID),
556-
}
557-
if o.LastModified != nil {
558-
obj.updated = *o.LastModified
550+
func newObjectFromHead(f *FS, name string, o *s3.HeadObjectOutput) *object {
551+
obj := &object{
552+
fs: f,
553+
name: name,
554+
bucket: f.bucket,
555+
cachepath: cloudstorage.CachePathObj(f.cachepath, name, f.ID),
556+
}
557+
if o.LastModified != nil {
558+
obj.updated = *o.LastModified
559+
}
560+
// metadata?
561+
obj.metadata, _ = convertMetaData(o.Metadata)
562+
return obj
559563
}
560-
// metadata?
561-
obj.metadata, _ = convertMetaData(o.Metadata)
562-
return obj
563-
}
564564
*/
565565
func (o *object) StorageSource() string {
566566
return StoreType

‎azure/store_test.go‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,24 @@ import (
55
"testing"
66

77
"github.com/araddon/gou"
8-
"github.com/stretchr/testify/assert"
8+
"github.com/stretchr/testify/require"
99

1010
"github.com/lytics/cloudstorage"
1111
"github.com/lytics/cloudstorage/azure"
1212
"github.com/lytics/cloudstorage/testutils"
1313
)
1414

1515
/*
16-
1716
# to use azure tests ensure you have exported
1817
1918
export AZURE_KEY="aaa"
2019
export AZURE_PROJECT="bbb"
2120
export AZURE_BUCKET="cloudstorageunittests"
22-
2321
*/
2422
var config = &cloudstorage.Config{
2523
Type: azure.StoreType,
2624
AuthMethod: azure.AuthKey,
2725
Bucket: os.Getenv("AZURE_BUCKET"),
28-
TmpDir: "/tmp/localcache/azure",
2926
Settings: make(gou.JsonHelper),
3027
}
3128

@@ -35,36 +32,37 @@ func TestConfig(t *testing.T) {
3532
t.Skip()
3633
return
3734
}
35+
3836
conf := &cloudstorage.Config{
3937
Type: azure.StoreType,
4038
Project: os.Getenv("AZURE_PROJECT"),
4139
Settings: make(gou.JsonHelper),
4240
}
4341
// Should error with empty config
4442
_, err := cloudstorage.NewStore(conf)
45-
assert.NotEqual(t, nil, err)
43+
require.Error(t, err)
4644

4745
conf.AuthMethod = azure.AuthKey
4846
conf.Settings[azure.ConfKeyAuthKey] = ""
4947
_, err = cloudstorage.NewStore(conf)
50-
assert.NotEqual(t, nil, err)
48+
require.Error(t, err)
5149

5250
conf.Settings[azure.ConfKeyAuthKey] = "bad"
5351
_, err = cloudstorage.NewStore(conf)
54-
assert.NotEqual(t, nil, err)
52+
require.Error(t, err)
5553

5654
conf.Settings[azure.ConfKeyAuthKey] = os.Getenv("AZURE_KEY")
5755
client, sess, err := azure.NewClient(conf)
58-
assert.Equal(t, nil, err)
59-
assert.NotEqual(t, nil, client)
56+
require.NoError(t, err)
57+
require.NotNil(t, client)
6058
conf.TmpDir = ""
6159
_, err = azure.NewStore(client, sess, conf)
62-
assert.NotEqual(t, nil, err)
60+
require.Error(t, err)
6361

6462
// Trying to find dir they don't have access to?
6563
conf.TmpDir = "/home/fake"
6664
_, err = cloudstorage.NewStore(conf)
67-
assert.NotEqual(t, nil, err)
65+
require.Error(t, err)
6866
}
6967

7068
func TestAll(t *testing.T) {
@@ -74,6 +72,9 @@ func TestAll(t *testing.T) {
7472
t.Skip()
7573
return
7674
}
75+
76+
config.TmpDir = t.TempDir()
77+
7778
config.Settings[azure.ConfKeyAuthKey] = os.Getenv("AZURE_KEY")
7879
store, err := cloudstorage.NewStore(config)
7980
if err != nil {
@@ -82,7 +83,7 @@ func TestAll(t *testing.T) {
8283
return
8384
}
8485
client := store.Client()
85-
assert.NotEqual(t, nil, client)
86+
require.NotNil(t, client)
8687

8788
testutils.RunTests(t, store, config)
8889
}

‎csbufio/reader_test.go‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"testing"
66

7+
"github.com/acomagu/bufpipe"
78
"github.com/stretchr/testify/require"
89
)
910

@@ -13,8 +14,9 @@ func TestReaderContextDone(t *testing.T) {
1314
ctx, cancel := context.WithCancel(context.Background())
1415
cancel()
1516

16-
m := memRWC([]byte("some-data"))
17-
rc := NewReader(ctx, &m)
17+
pr, pw := bufpipe.New([]byte("some-data"))
18+
pw.Close()
19+
rc := NewReader(ctx, pr)
1820

1921
var p []byte
2022
n, err := rc.Read(p)
@@ -25,23 +27,3 @@ func TestReaderContextDone(t *testing.T) {
2527
err = rc.Close()
2628
require.ErrorIs(t, err, context.Canceled)
2729
}
28-
29-
type memRWC []byte
30-
31-
func (m memRWC) Read(p []byte) (int, error) {
32-
n := len(p)
33-
if n > len(m) {
34-
n = len(m)
35-
}
36-
copy(p, m)
37-
return n, nil
38-
}
39-
40-
func (m *memRWC) Write(p []byte) (int, error) {
41-
*m = append(*m, p...)
42-
return len(p), nil
43-
}
44-
45-
func (m memRWC) Close() error {
46-
return nil
47-
}

‎csbufio/writer_test.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ package csbufio
22

33
import (
44
"context"
5+
"io"
56
"testing"
67

8+
"github.com/acomagu/bufpipe"
79
"github.com/stretchr/testify/require"
810
)
911

@@ -13,13 +15,18 @@ func TestWriterContextDone(t *testing.T) {
1315
ctx, cancel := context.WithCancel(context.Background())
1416
cancel()
1517

16-
var m memRWC
17-
wc := NewWriter(ctx, &m)
18+
pr, pw := bufpipe.New(nil)
19+
wc := NewWriter(ctx, pw)
1820

1921
n, err := wc.Write([]byte("some-data"))
2022
require.ErrorIs(t, err, context.Canceled)
2123
require.Equal(t, 0, n)
22-
require.Len(t, m, 0)
24+
err = pw.Close()
25+
require.NoError(t, err)
26+
27+
b, err := io.ReadAll(pr)
28+
require.NoError(t, err, "error reading")
29+
require.Equal(t, 0, len(b), "")
2330

2431
err = wc.Close()
2532
require.ErrorIs(t, err, context.Canceled)

‎file_helper_test.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@ package cloudstorage
33
import (
44
"testing"
55

6-
"github.com/stretchr/testify/assert"
6+
"github.com/stretchr/testify/require"
77
)
88

99
func TestETAG(t *testing.T) {
10-
assert.Equal(t, "hello", CleanETag("hello"))
11-
assert.Equal(t, "hello", CleanETag(`"hello"`))
12-
assert.Equal(t, "hello", CleanETag(`\"hello\"`))
13-
assert.Equal(t, "hello", CleanETag("\"hello\""))
10+
require.Equal(t, "hello", CleanETag("hello"))
11+
require.Equal(t, "hello", CleanETag(`"hello"`))
12+
require.Equal(t, "hello", CleanETag(`\"hello\"`))
13+
require.Equal(t, "hello", CleanETag("\"hello\""))
1414
}
1515
func TestContentType(t *testing.T) {
16-
assert.Equal(t, "text/csv; charset=utf-8", ContentType("data.csv"))
17-
assert.Equal(t, "application/json", ContentType("data.json"))
18-
assert.Equal(t, "application/octet-stream", ContentType("data.unknown"))
16+
require.Equal(t, "text/csv; charset=utf-8", ContentType("data.csv"))
17+
require.Equal(t, "application/json", ContentType("data.json"))
18+
require.Equal(t, "application/octet-stream", ContentType("data.unknown"))
1919
}

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ require (
1717
google.golang.org/api v0.103.0
1818
)
1919

20+
require github.com/acomagu/bufpipe v1.0.4
21+
2022
require (
2123
cloud.google.com/go v0.105.0 // indirect
2224
cloud.google.com/go/compute v1.12.1 // indirect

0 commit comments

Comments
 (0)