Skip to content
This repository was archived by the owner on Jul 24, 2024. It is now read-only.

Commit 9adc69b

Browse files
ti-srebot3pointer
ti-srebot
and
3pointer
authored
gcs: compatible with old gcs bug (#677) (#689)
* cherry pick #677 to release-5.0-rc Signed-off-by: ti-srebot <ti-srebot@pingcap.com> * fix build Co-authored-by: 3pointer <luancheng@pingcap.com>
1 parent 0329712 commit 9adc69b

11 files changed

Lines changed: 277 additions & 21 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ bins:
8484
@which bin/tiflash
8585
@which bin/libtiflash_proxy.so
8686
@which bin/cdc
87+
@which bin/fake-gcs-server
8788
if [ ! -d bin/flash_cluster_manager ]; then echo "flash_cluster_manager not exist"; exit 1; fi
8889

8990
tools:

‎go.mod1‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,5 @@ require (
3636
google.golang.org/api v0.22.0
3737
google.golang.org/grpc v1.27.1
3838
)
39+
40+
replace cloud.google.com/go/storage => github.com/3pointer/google-cloud-go/storage v1.6.1-0.20210108125931-b59bfa0720b2

‎go.sum1‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK
77
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
88
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
99
cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY=
10-
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
1110
cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
1211
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
1312
cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8=
@@ -25,12 +24,10 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+
2524
cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680=
2625
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
2726
cloud.google.com/go/spanner v1.1.0/go.mod h1:TzTaF9l2ZY2CIetNvVpUu6ZQy8YEOtzB6ICa5EwYjL0=
28-
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
29-
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
30-
cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8=
31-
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
3227
collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
3328
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
29+
github.com/3pointer/google-cloud-go/storage v1.6.1-0.20210108125931-b59bfa0720b2 h1:Bnm+q0FAI1AXoPX3cPIi2/a4sg2J2f/D0yPr+7EMPg0=
30+
github.com/3pointer/google-cloud-go/storage v1.6.1-0.20210108125931-b59bfa0720b2/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
3431
github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
3532
github.com/AstroProfundis/sysinfo v0.0.0-20200423033635-f6f7687215fd/go.mod h1:4m15VhW6ZffaqJNAK/HtN3Qvf97aCe1T39u3UXaA2lA=
3633
github.com/AstroProfundis/tabby v1.1.0-color/go.mod h1:Wcm+uinH1saEOFGLK2LdY37lAOts8HLevz64Y3y3M3Q=
@@ -784,7 +781,6 @@ github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfP
784781
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
785782
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
786783
github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
787-
github.com/shirou/gopsutil v2.20.3+incompatible h1:0JVooMPsT7A7HqEYdydp/OfjSOYSjhXV7w1hkKj/NPQ=
788784
github.com/shirou/gopsutil v2.20.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
789785
github.com/shirou/gopsutil v2.20.6+incompatible h1:P37G9YH8M4vqkKcwBosp+URN5O8Tay67D2MbR361ioY=
790786
github.com/shirou/gopsutil v2.20.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
@@ -1119,7 +1115,6 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7w
11191115
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
11201116
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
11211117
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
1122-
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
11231118
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
11241119
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
11251120
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1193,7 +1188,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
11931188
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
11941189
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
11951190
golang.org/x/tools v0.0.0-20191210221141-98df12377212/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
1196-
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
11971191
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
11981192
golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
11991193
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -1255,7 +1249,6 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx
12551249
google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
12561250
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
12571251
google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
1258-
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
12591252
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
12601253
google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
12611254
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=

‎pkg/mock/mock_cluster.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,10 @@ func NewCluster() (*Cluster, error) {
5959

6060
mvccStore := mocktikv.MustNewMVCCStore()
6161
client, cluster, pdClient, err := mocktikv.NewTiKVAndPDClient("")
62-
mocktikv.BootstrapWithSingleStore(cluster)
63-
6462
if err != nil {
6563
return nil, errors.Trace(err)
6664
}
65+
mocktikv.BootstrapWithSingleStore(cluster)
6766
storage, err := tikv.NewTestTiKVStore(client, pdClient, nil, nil, 0)
6867
if err != nil {
6968
return nil, errors.Trace(err)

‎pkg/storage/gcs.go‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,16 @@ import (
77
"io"
88
"io/ioutil"
99
"path"
10+
"strings"
1011

1112
"cloud.google.com/go/storage"
1213
"github.com/pingcap/errors"
1314
"github.com/pingcap/kvproto/pkg/backup"
15+
"github.com/pingcap/log"
1416
"github.com/spf13/pflag"
17+
"go.uber.org/zap"
1518
"golang.org/x/oauth2/google"
19+
"google.golang.org/api/iterator"
1620
"google.golang.org/api/option"
1721

1822
berrors "github.com/pingcap/br/pkg/errors"
@@ -111,8 +115,15 @@ func (s *gcsStorage) Read(ctx context.Context, name string) ([]byte, error) {
111115
}
112116
defer rc.Close()
113117

114-
b := make([]byte, rc.Attrs.Size)
115-
_, err = io.ReadFull(rc, b)
118+
size := rc.Attrs.Size
119+
var b []byte
120+
if size < 0 {
121+
// happened when using fake-gcs-server in integration test
122+
b, err = ioutil.ReadAll(rc)
123+
} else {
124+
b = make([]byte, size)
125+
_, err = io.ReadFull(rc, b)
126+
}
116127
return b, errors.Trace(err)
117128
}
118129

@@ -171,7 +182,9 @@ func newGCSStorage(ctx context.Context, gcs *backup.GCS, opts *ExternalStorageOp
171182
"You should provide '--gcs.credentials_file' when '--send-credentials-to-tikv' is true")
172183
}
173184
}
174-
clientOps = append(clientOps, option.WithCredentials(creds))
185+
if creds != nil {
186+
clientOps = append(clientOps, option.WithCredentials(creds))
187+
}
175188
} else {
176189
clientOps = append(clientOps, option.WithCredentialsJSON([]byte(gcs.GetCredentialsBlob())))
177190
}
@@ -193,6 +206,18 @@ func newGCSStorage(ctx context.Context, gcs *backup.GCS, opts *ExternalStorageOp
193206
}
194207

195208
bucket := client.Bucket(gcs.Bucket)
209+
// check whether it's a bug before #647, to solve case #2
210+
// If the storage is set as gcs://bucket/prefix/,
211+
// the backupmeta is written correctly to gcs://bucket/prefix/backupmeta,
212+
// but the SSTs are written wrongly to gcs://bucket/prefix//*.sst (note the extra slash).
213+
// see details about case 2 at https://github.com/pingcap/br/issues/675#issuecomment-753780742
214+
sstInPrefix := hasSSTFiles(ctx, bucket, gcs.Prefix)
215+
sstInPrefixSlash := hasSSTFiles(ctx, bucket, gcs.Prefix+"//")
216+
if sstInPrefixSlash && !sstInPrefix {
217+
// This is a old bug, but we must make it compatible.
218+
// so we need find sst in slash directory
219+
gcs.Prefix += "//"
220+
}
196221
if !opts.SkipCheckPath {
197222
// check bucket exists
198223
_, err = bucket.Attrs(ctx)
@@ -202,3 +227,24 @@ func newGCSStorage(ctx context.Context, gcs *backup.GCS, opts *ExternalStorageOp
202227
}
203228
return &gcsStorage{gcs: gcs, bucket: bucket}, nil
204229
}
230+
231+
func hasSSTFiles(ctx context.Context, bucket *storage.BucketHandle, prefix string) bool {
232+
query := storage.Query{Prefix: prefix}
233+
_ = query.SetAttrSelection([]string{"Name"})
234+
it := bucket.Objects(ctx, &query)
235+
for {
236+
attrs, err := it.Next()
237+
if err == iterator.Done { // nolint:errorlint
238+
break
239+
}
240+
if err != nil {
241+
log.Warn("failed to list objects on gcs, will use default value for `prefix`", zap.Error(err))
242+
break
243+
}
244+
if strings.HasSuffix(attrs.Name, ".sst") {
245+
log.Info("sst file found in prefix slash", zap.String("file", attrs.Name))
246+
return true
247+
}
248+
}
249+
return false
250+
}

‎pkg/storage/parse.go‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,26 @@ type BackendOptions struct {
2222
GCS GCSBackendOptions `json:"gcs" toml:"gcs"`
2323
}
2424

25+
// ParseRawURL parse raw url to url object.
26+
func ParseRawURL(rawURL string) (*url.URL, error) {
27+
// https://github.com/pingcap/br/issues/603
28+
// In aws the secret key may contain '/+=' and '+' has a special meaning in URL.
29+
// Replace "+" by "%2B" here to avoid this problem.
30+
rawURL = strings.ReplaceAll(rawURL, "+", "%2B")
31+
u, err := url.Parse(rawURL)
32+
if err != nil {
33+
return nil, errors.Trace(err)
34+
}
35+
return u, nil
36+
}
37+
2538
// ParseBackend constructs a structured backend description from the
2639
// storage URL.
2740
func ParseBackend(rawURL string, options *BackendOptions) (*backup.StorageBackend, error) {
2841
if len(rawURL) == 0 {
2942
return nil, errors.Annotate(berrors.ErrStorageInvalidConfig, "empty store is not allowed")
3043
}
31-
32-
// https://github.com/pingcap/br/issues/603
33-
// In aws the secret key may contain '/+=' and '+' has a special meaning in URL.
34-
// Replace "+" by "%2B" here to avoid this problem.
35-
rawURL = strings.ReplaceAll(rawURL, "+", "%2B")
36-
u, err := url.Parse(rawURL)
44+
u, err := ParseRawURL(rawURL)
3745
if err != nil {
3846
return nil, errors.Trace(err)
3947
}

‎pkg/task/common.go‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ import (
77
"crypto/tls"
88
"fmt"
99
"net/url"
10+
"path"
1011
"strings"
1112
"time"
1213

14+
gcs "cloud.google.com/go/storage"
1315
"github.com/gogo/protobuf/proto"
1416
"github.com/pingcap/errors"
1517
"github.com/pingcap/kvproto/pkg/backup"
@@ -394,7 +396,26 @@ func ReadBackupMeta(
394396
}
395397
metaData, err := s.Read(ctx, fileName)
396398
if err != nil {
397-
return nil, nil, nil, errors.Annotate(err, "load backupmeta failed")
399+
if gcsObjectNotFound(err) {
400+
// change gcs://bucket/abc/def to gcs://bucket/abc and read defbackupmeta
401+
oldPrefix := u.GetGcs().GetPrefix()
402+
newPrefix, file := path.Split(oldPrefix)
403+
newFileName := file + fileName
404+
u.GetGcs().Prefix = newPrefix
405+
s, err = storage.Create(ctx, u, cfg.SendCreds)
406+
if err != nil {
407+
return nil, nil, nil, errors.Trace(err)
408+
}
409+
log.Info("retry load metadata in gcs", zap.String("newPrefix", newPrefix), zap.String("newFileName", newFileName))
410+
metaData, err = s.Read(ctx, newFileName)
411+
if err != nil {
412+
return nil, nil, nil, errors.Trace(err)
413+
}
414+
// reset prefix for tikv download sst file correctly.
415+
u.GetGcs().Prefix = oldPrefix
416+
} else {
417+
return nil, nil, nil, errors.Annotate(err, "load backupmeta failed")
418+
}
398419
}
399420
backupMeta := &backup.BackupMeta{}
400421
if err = proto.Unmarshal(metaData, backupMeta); err != nil {
@@ -466,3 +487,12 @@ func normalizePDURL(pd string, useTLS bool) (string, error) {
466487
}
467488
return pd, nil
468489
}
490+
491+
// check whether it's a bug before #647, to solve case #1
492+
// If the storage is set as gcs://bucket/prefix,
493+
// the SSTs are written correctly to gcs://bucket/prefix/*.sst
494+
// but the backupmeta is written wrongly to gcs://bucket/prefixbackupmeta.
495+
// see details https://github.com/pingcap/br/issues/675#issuecomment-753780742
496+
func gcsObjectNotFound(err error) bool {
497+
return errors.Cause(err) == gcs.ErrObjectNotExist // nolint:errorlint
498+
}

‎tests/br_gcs/oauth.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env python3
2+
3+
from flask import Flask
4+
5+
app = Flask(__name__)
6+
7+
@app.route('/oauth/token', methods=['GET', 'POST'])
8+
def oauth():
9+
return '{"access_token": "ok", "token_type":"service_account", "expires_in":3600}'

0 commit comments

Comments
 (0)