From 970341a12a7d0ea218017d00d39f6a5ebe074a3e Mon Sep 17 00:00:00 2001 From: ktsivkov Date: Mon, 20 Mar 2023 15:01:08 +0100 Subject: [PATCH] Implemented easer feature to reduce the DB load --- callbacks/helper.go | 57 +++++++++++ callbacks/helper_test.go | 209 +++++++++++++++++++++++++++++++++++++++ callbacks/query.go | 41 +++++--- ease.go | 10 ++ gorm.go | 30 ++++++ 5 files changed, 334 insertions(+), 13 deletions(-) create mode 100644 callbacks/helper_test.go create mode 100644 ease.go diff --git a/callbacks/helper.go b/callbacks/helper.go index ae9fd8c5..32326c58 100644 --- a/callbacks/helper.go +++ b/callbacks/helper.go @@ -1,11 +1,14 @@ package callbacks import ( + "errors" + "fmt" "reflect" "sort" "gorm.io/gorm" "gorm.io/gorm/clause" + "gorm.io/gorm/schema" ) // ConvertMapToValuesForCreate convert map to values @@ -150,3 +153,57 @@ func loadOrStoreVisitMap(visitMap *visitMap, v reflect.Value) (loaded bool) { return } + +func deepCopy(src, dst interface{}) error { + srcVal := reflect.ValueOf(src) + dstVal := reflect.ValueOf(dst) + + if srcVal.Kind() == reflect.Ptr { + srcVal = srcVal.Elem() + } + + if srcVal.Type() != dstVal.Elem().Type() { + return errors.New("src and dst must be of the same type") + } + + return copyValue(srcVal, dstVal.Elem()) +} + +func copyValue(src, dst reflect.Value) error { + switch src.Kind() { + case reflect.Ptr: + src = src.Elem() + dst.Set(reflect.New(src.Type())) + copyValue(src, dst.Elem()) + + case reflect.Struct: + for i := 0; i < src.NumField(); i++ { + if src.Type().Field(i).PkgPath != "" { + return fmt.Errorf("%w: %+v", schema.ErrUnsupportedDataType, src.Type().Field(i).Name) + } + copyValue(src.Field(i), dst.Field(i)) + } + + case reflect.Slice: + newSlice := reflect.MakeSlice(src.Type(), src.Len(), src.Cap()) + for i := 0; i < src.Len(); i++ { + copyValue(src.Index(i), newSlice.Index(i)) + } + dst.Set(newSlice) + + case reflect.Map: + newMap := reflect.MakeMapWithSize(src.Type(), src.Len()) + for _, key := range src.MapKeys() { + value := src.MapIndex(key) + newValue := reflect.New(value.Type()).Elem() + copyValue(value, newValue) + newMap.SetMapIndex(key, newValue) + } + dst.Set(newMap) + + default: + dst.Set(src) + } + + return nil +} diff --git a/callbacks/helper_test.go b/callbacks/helper_test.go new file mode 100644 index 00000000..5eebcc85 --- /dev/null +++ b/callbacks/helper_test.go @@ -0,0 +1,209 @@ +package callbacks + +import ( + "reflect" + "testing" +) + +type unsupportedMockStruct struct { + ExportedField string + unexportedField string + ExportedSliceField []string + unexportedSliceField []string + ExportedMapField map[string]string + unexportedMapField map[string]string +} + +type supportedMockStruct struct { + ExportedField string + ExportedSliceField []string + ExportedMapField map[string]string +} + +func TestDeepCopy(t *testing.T) { + t.Run("struct", func(t *testing.T) { + t.Run("supported", func(t *testing.T) { + srcStruct := supportedMockStruct{ + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + } + dstStruct := supportedMockStruct{} + + err := deepCopy(srcStruct, &dstStruct) + + if err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcStruct, dstStruct) { + t.Errorf("deepCopy failed to copy structure: got %+v, want %+v", dstStruct, srcStruct) + } + }) + t.Run("unsupported", func(t *testing.T) { + srcStruct := unsupportedMockStruct{ + ExportedField: "exported field", + unexportedField: "unexported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + unexportedSliceField: []string{"1st elem of an unexported slice field", "2nd elem of an unexported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + unexportedMapField: map[string]string{ + "key1": "unexported map elem", + "key2": "unexported map elem", + }, + } + dstStruct := unsupportedMockStruct{} + + err := deepCopy(srcStruct, &dstStruct) + + if err == nil { + t.Error("deepCopy was expected to fail copying an structure with unexported fields") + } + }) + }) + + t.Run("map", func(t *testing.T) { + t.Run("map[string]string", func(t *testing.T) { + srcMap := map[string]string{ + "key1": "value1", + "key2": "value2", + } + dstMap := make(map[string]string) + + err := deepCopy(srcMap, &dstMap) + + if err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcMap, dstMap) { + t.Errorf("deepCopy failed to copy map: got %+v, want %+v", dstMap, srcMap) + } + }) + + t.Run("map[string]struct", func(t *testing.T) { + srcMap := map[string]supportedMockStruct{ + "key1": { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + "key2": { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + } + dstMap := make(map[string]supportedMockStruct) + + err := deepCopy(srcMap, &dstMap) + + if err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcMap, dstMap) { + t.Errorf("deepCopy failed to copy map: got %+v, want %+v", dstMap, srcMap) + } + }) + }) + + t.Run("slice", func(t *testing.T) { + t.Run("[]string", func(t *testing.T) { + srcSlice := []string{"A", "B", "C"} + dstSlice := make([]string, len(srcSlice)) + + err := deepCopy(srcSlice, &dstSlice) + + if err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcSlice, dstSlice) { + t.Errorf("deepCopy failed to copy slice: got %+v, want %+v", dstSlice, srcSlice) + } + }) + t.Run("[]struct", func(t *testing.T) { + srcSlice := []supportedMockStruct{ + { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + } + dstSlice := make([]supportedMockStruct, len(srcSlice)) + + err := deepCopy(srcSlice, &dstSlice) + + if err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcSlice, dstSlice) { + t.Errorf("deepCopy failed to copy slice: got %+v, want %+v", dstSlice, srcSlice) + } + }) + }) + + t.Run("pointer", func(t *testing.T) { + srcStruct := &supportedMockStruct{ + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + } + dstStruct := &supportedMockStruct{} + + err := deepCopy(srcStruct, dstStruct) + + if err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcStruct, dstStruct) { + t.Errorf("deepCopy failed to copy structure: got %+v, want %+v", dstStruct, srcStruct) + } + }) + + t.Run("mismatched", func(t *testing.T) { + src := "a string" + dst := 123 + + err := deepCopy(src, &dst) + + if err == nil { + t.Error("deepCopy did not return an error when provided mismatched types") + } + }) +} diff --git a/callbacks/query.go b/callbacks/query.go index c87f17bc..990bdce4 100644 --- a/callbacks/query.go +++ b/callbacks/query.go @@ -2,31 +2,46 @@ package callbacks import ( "fmt" - "reflect" - "sort" - "strings" - "gorm.io/gorm" "gorm.io/gorm/clause" "gorm.io/gorm/schema" "gorm.io/gorm/utils" + "reflect" + "sort" + "strings" + "time" ) func Query(db *gorm.DB) { if db.Error == nil { BuildQuerySQL(db) - if !db.DryRun && db.Error == nil { - rows, err := db.Statement.ConnPool.QueryContext(db.Statement.Context, db.Statement.SQL.String(), db.Statement.Vars...) - if err != nil { - db.AddError(err) - return + var _query = func(db *gorm.DB) { + time.Sleep(5 * time.Second) + if !db.DryRun && db.Error == nil { + rows, err := db.Statement.ConnPool.QueryContext(db.Statement.Context, db.Statement.SQL.String(), db.Statement.Vars...) + if err != nil { + db.AddError(err) + return + } + defer func() { + db.AddError(rows.Close()) + }() + gorm.Scan(rows, db, 0) } - defer func() { - db.AddError(rows.Close()) - }() - gorm.Scan(rows, db, 0) } + + if db.Config.Ease { + eqDb := db.Ease(_query) + if db.Statement.Dest != eqDb.Statement.Dest { + if err := deepCopy(eqDb.Statement.Dest, db.Statement.Dest); err != nil { + db.AddError(err) + } + } + return + } + + _query(db) } } diff --git a/ease.go b/ease.go new file mode 100644 index 00000000..1eb56f39 --- /dev/null +++ b/ease.go @@ -0,0 +1,10 @@ +package gorm + +import ( + "sync" +) + +type easedTask struct { + wg *sync.WaitGroup + db *DB +} diff --git a/gorm.go b/gorm.go index 9a70c3d2..c3db1297 100644 --- a/gorm.go +++ b/gorm.go @@ -56,9 +56,12 @@ type Config struct { Dialector // Plugins registered plugins Plugins map[string]Plugin + // Ease database load by grouping identical queries + Ease bool callbacks *callbacks cacheStore *sync.Map + easeQueue *sync.Map } // Apply update config to new config @@ -167,6 +170,10 @@ func Open(dialector Dialector, opts ...Option) (db *DB, err error) { config.cacheStore = &sync.Map{} } + if config.Ease && config.easeQueue == nil { + config.easeQueue = &sync.Map{} + } + db = &DB{Config: config, clone: 1} db.callbacks = initializeCallbacks(db) @@ -484,3 +491,26 @@ func (db *DB) ToSQL(queryFn func(tx *DB) *DB) string { return db.Dialector.Explain(stmt.SQL.String(), stmt.Vars...) } + +func (db *DB) Ease(cb func(*DB)) *DB { + hash := db.Statement.SQL.String() + + eq := &easedTask{ + wg: &sync.WaitGroup{}, + db: db, + } + eq.wg.Add(1) + + var runner, ok = db.easeQueue.LoadOrStore(hash, eq) + et := runner.(*easedTask) + + if !ok { + cb(db) + et.wg.Done() + db.easeQueue.Delete(hash) + return db + } + + et.wg.Wait() + return et.db +}