diff --git a/CHANGELOG.md b/CHANGELOG.md index bda708d31..d19c426b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v1.10.1 +FEATURES +* [#1440](https://github.com/bnb-chain/greenfield-storage-provider/pull/1440) feat: add index to permission table and add piece store no such key error as 404 code + ## v1.10.0 BUGFIXES * [#1435](https://github.com/bnb-chain/greenfield-storage-provider/pull/1435) fix: gc objects diff --git a/go.mod b/go.mod index af0b8d70e..b7524a5dc 100644 --- a/go.mod +++ b/go.mod @@ -286,7 +286,7 @@ require ( google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.33.0 gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -308,7 +308,7 @@ replace ( github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.9.0 github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1 - github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20240604033531-028f2cc8f76d + github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20241213040233-b35dbdd3eccc github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 ) diff --git a/go.sum b/go.sum index 70d76144d..c5ef24de2 100644 --- a/go.sum +++ b/go.sum @@ -192,8 +192,8 @@ github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965b github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965bc8/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM= github.com/bnb-chain/greenfield-iavl v0.20.1 h1:y3L64GU99otNp27/xLVBTDbv4eroR6CzoYz0rbaVotM= github.com/bnb-chain/greenfield-iavl v0.20.1/go.mod h1:oLksTs8dfh7DYIKBro7hbRQ+ewls7ghJ27pIXlbEXyI= -github.com/bnb-chain/juno/v4 v4.0.0-20240604033531-028f2cc8f76d h1:a5Q944ZvZK4K3lDkbIqweZd1CqEE8gmxph+wpTQzE3k= -github.com/bnb-chain/juno/v4 v4.0.0-20240604033531-028f2cc8f76d/go.mod h1:p+KkPIHURpqUJOdXanhhYgZpTLQxGZWkrAwtUaOuPlM= +github.com/bnb-chain/juno/v4 v4.0.0-20241213040233-b35dbdd3eccc h1:+dDQk5ayB51Qgw5hZOzAkqJTyX6PDQFEj3aDEWmCLss= +github.com/bnb-chain/juno/v4 v4.0.0-20241213040233-b35dbdd3eccc/go.mod h1:p+KkPIHURpqUJOdXanhhYgZpTLQxGZWkrAwtUaOuPlM= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= diff --git a/modular/blocksyncer/modules/permission/module.go b/modular/blocksyncer/modules/permission/module.go index dca39e5bd..44a897637 100644 --- a/modular/blocksyncer/modules/permission/module.go +++ b/modular/blocksyncer/modules/permission/module.go @@ -63,5 +63,5 @@ func (m *Module) PrepareTables() error { // AutoMigrate implements func (m *Module) AutoMigrate() error { - return m.db.AutoMigrate(context.TODO(), []schema.Tabler{&models.Statements{}}) + return m.db.AutoMigrate(context.TODO(), []schema.Tabler{&models.Permission{}, &models.Statements{}}) } diff --git a/modular/downloader/download_task.go b/modular/downloader/download_task.go index 4bc0e8734..2731597e0 100644 --- a/modular/downloader/download_task.go +++ b/modular/downloader/download_task.go @@ -4,9 +4,11 @@ import ( "context" "errors" "net/http" + "strings" "sync/atomic" "time" + "github.com/aws/aws-sdk-go/service/s3" "gorm.io/gorm" "github.com/bnb-chain/greenfield-storage-provider/base/types/gfsperrors" @@ -39,6 +41,10 @@ func ErrPieceStoreWithDetail(detail string) *gfsperrors.GfSpError { return gfsperrors.Register(module.ReceiveModularName, http.StatusInternalServerError, 85101, detail) } +func ErrPieceStoreNoSuchKeyWithDetail(detail string) *gfsperrors.GfSpError { + return gfsperrors.Register(module.ReceiveModularName, http.StatusNotFound, 85102, detail) +} + func ErrGfSpDBWithDetail(detail string) *gfsperrors.GfSpError { return gfsperrors.Register(module.ReceiveModularName, http.StatusInternalServerError, 85201, detail) } @@ -175,8 +181,12 @@ func (d *DownloadModular) HandleDownloadObjectTask(ctx context.Context, download int64(pInfo.Offset), int64(pInfo.Length)) if getPieceErr != nil { log.CtxErrorw(ctx, "failed to get piece data from piece store", "task_info", downloadObjectTask.Info(), "piece_info", pInfo, "error", getPieceErr) - err = ErrPieceStoreWithDetail("failed to get piece data from piece store, error: " + getPieceErr.Error()) - return nil, err + pieceStoreErrDetail := "failed to get piece data from piece store, task_info: " + downloadObjectTask.Info() + ", error: " + err.Error() + if isErrNoSuchKey(err) { + return nil, ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail) + } else { + return nil, ErrPieceStoreWithDetail(pieceStoreErrDetail) + } } d.pieceCache.Add(key, piece) data = append(data, piece...) @@ -389,7 +399,11 @@ func (d *DownloadModular) HandleDownloadPieceTask(ctx context.Context, downloadP int64(downloadPieceTask.GetPieceOffset()), int64(downloadPieceTask.GetPieceLength())); err != nil { metrics.PerfGetObjectTimeHistogram.WithLabelValues("get_object_put_piece_time").Observe(time.Since(putPieceTime).Seconds()) log.CtxErrorw(ctx, "failed to get piece data from piece store", "task_info", downloadPieceTask.Info(), "error", err) - return nil, ErrPieceStoreWithDetail("failed to get piece data from piece store, task_info: " + downloadPieceTask.Info() + ", error: " + err.Error()) + pieceStoreErrDetail := "failed to get piece data from piece store, task_info: " + downloadPieceTask.Info() + ", error: " + err.Error() + if isErrNoSuchKey(err) { + return nil, ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail) + } + return nil, ErrPieceStoreWithDetail(pieceStoreErrDetail) } metrics.PerfGetObjectTimeHistogram.WithLabelValues("get_object_put_piece_time").Observe(time.Since(putPieceTime).Seconds()) return pieceData, nil @@ -462,7 +476,12 @@ func (d *DownloadModular) HandleChallengePiece(ctx context.Context, challengePie metrics.PerfChallengeTimeHistogram.WithLabelValues("challenge_get_piece_time").Observe(time.Since(getPieceTime).Seconds()) if err != nil { log.CtxErrorw(ctx, "failed to get piece data", "task", challengePieceTask, "error", err) - return nil, nil, nil, ErrPieceStoreWithDetail("failed to get piece data, error: " + err.Error()) + pieceStoreErrDetail := "failed to get piece data, task: " + challengePieceTask.Info() + ", error: " + err.Error() + if isErrNoSuchKey(err) { + return nil, nil, nil, ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail) + } else { + return nil, nil, nil, ErrPieceStoreWithDetail(pieceStoreErrDetail) + } } return integrity.IntegrityChecksum, integrity.PieceChecksumList, data, nil @@ -474,3 +493,8 @@ func (d *DownloadModular) PostChallengePiece(context.Context, task.ChallengePiec func (d *DownloadModular) QueryTasks(context.Context, task.TKey) ([]task.Task, error) { return nil, nil } + +func isErrNoSuchKey(err error) bool { + msg := err.Error() + return strings.Contains(msg, s3.ErrCodeNoSuchKey) +} diff --git a/modular/gater/admin_handler.go b/modular/gater/admin_handler.go index 8434e0462..757dd2388 100644 --- a/modular/gater/admin_handler.go +++ b/modular/gater/admin_handler.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/service/s3" sdktypes "github.com/cosmos/cosmos-sdk/types" commonhash "github.com/bnb-chain/greenfield-common/go/hash" @@ -1001,7 +1002,12 @@ func (g *GateModular) getRecoverPiece(ctx context.Context, objectInfo *storagety pieceData, err := g.baseApp.GfSpClient().GetPiece(ctx, pieceTask) if err != nil { log.CtxErrorw(ctx, "failed to download piece", "error", err) - return nil, downloader.ErrPieceStoreWithDetail("failed to download piece, error: " + err.Error()) + pieceStoreErrDetail := "failed to download piece, error: " + err.Error() + if isErrNoSuchKey(err) { + return nil, downloader.ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail) + } else { + return nil, downloader.ErrPieceStoreWithDetail(pieceStoreErrDetail) + } } return pieceData, nil @@ -1091,7 +1097,12 @@ func (g *GateModular) getRecoverSegment(ctx context.Context, objectInfo *storage pieceData, err := g.baseApp.GfSpClient().GetPiece(ctx, pieceTask) if err != nil { log.CtxErrorw(ctx, "failed to download piece", "error", err) - return nil, downloader.ErrPieceStoreWithDetail("failed to download piece, error: " + err.Error()) + pieceStoreErrDetail := "failed to download piece, error: " + err.Error() + if isErrNoSuchKey(err) { + return nil, downloader.ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail) + } else { + return nil, downloader.ErrPieceStoreWithDetail(pieceStoreErrDetail) + } } return pieceData, nil } @@ -1117,3 +1128,8 @@ func (g *GateModular) getRecoverSegment(ctx context.Context, objectInfo *storage return ecData[redundancyIdx], nil } + +func isErrNoSuchKey(err error) bool { + msg := err.Error() + return strings.Contains(msg, s3.ErrCodeNoSuchKey) +} diff --git a/modular/gater/object_handler.go b/modular/gater/object_handler.go index eed9f2651..3f7ddd265 100644 --- a/modular/gater/object_handler.go +++ b/modular/gater/object_handler.go @@ -663,7 +663,7 @@ func (g *GateModular) downloadObject(w http.ResponseWriter, reqCtx *RequestConte log.CtxErrorw(reqCtx.Context(), "failed to download piece", "error", err) downloaderErr := gfsperrors.MakeGfSpError(err) // if it is the first piece and the quota db is not updated, no extra data need to updated - if idx >= 1 || (idx == 0 && downloaderErr.GetInnerCode() == 85101) { + if idx >= 1 || (idx == 0 && (downloaderErr.GetInnerCode() == 85101 || downloaderErr.GetInnerCode() == 85102)) { extraQuota = downloadSize - consumedQuota } return err