Implemented easer feature to reduce the DB load
This commit is contained in:
ktsivkov
parent
cc2d46e5be
commit
970341a12a
@ -1,11 +1,14 @@
|
||||
package callbacks
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"gorm.io/gorm/schema"
|
||||
)
|
||||
|
||||
// ConvertMapToValuesForCreate convert map to values
|
||||
@ -150,3 +153,57 @@ func loadOrStoreVisitMap(visitMap *visitMap, v reflect.Value) (loaded bool) {
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func deepCopy(src, dst interface{}) error {
|
||||
srcVal := reflect.ValueOf(src)
|
||||
dstVal := reflect.ValueOf(dst)
|
||||
|
||||
if srcVal.Kind() == reflect.Ptr {
|
||||
srcVal = srcVal.Elem()
|
||||
}
|
||||
|
||||
if srcVal.Type() != dstVal.Elem().Type() {
|
||||
return errors.New("src and dst must be of the same type")
|
||||
}
|
||||
|
||||
return copyValue(srcVal, dstVal.Elem())
|
||||
}
|
||||
|
||||
func copyValue(src, dst reflect.Value) error {
|
||||
switch src.Kind() {
|
||||
case reflect.Ptr:
|
||||
src = src.Elem()
|
||||
dst.Set(reflect.New(src.Type()))
|
||||
copyValue(src, dst.Elem())
|
||||
|
||||
case reflect.Struct:
|
||||
for i := 0; i < src.NumField(); i++ {
|
||||
if src.Type().Field(i).PkgPath != "" {
|
||||
return fmt.Errorf("%w: %+v", schema.ErrUnsupportedDataType, src.Type().Field(i).Name)
|
||||
}
|
||||
copyValue(src.Field(i), dst.Field(i))
|
||||
}
|
||||
|
||||
case reflect.Slice:
|
||||
newSlice := reflect.MakeSlice(src.Type(), src.Len(), src.Cap())
|
||||
for i := 0; i < src.Len(); i++ {
|
||||
copyValue(src.Index(i), newSlice.Index(i))
|
||||
}
|
||||
dst.Set(newSlice)
|
||||
|
||||
case reflect.Map:
|
||||
newMap := reflect.MakeMapWithSize(src.Type(), src.Len())
|
||||
for _, key := range src.MapKeys() {
|
||||
value := src.MapIndex(key)
|
||||
newValue := reflect.New(value.Type()).Elem()
|
||||
copyValue(value, newValue)
|
||||
newMap.SetMapIndex(key, newValue)
|
||||
}
|
||||
dst.Set(newMap)
|
||||
|
||||
default:
|
||||
dst.Set(src)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
209
callbacks/helper_test.go
Normal file
209
callbacks/helper_test.go
Normal file
@ -0,0 +1,209 @@
|
||||
package callbacks
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type unsupportedMockStruct struct {
|
||||
ExportedField string
|
||||
unexportedField string
|
||||
ExportedSliceField []string
|
||||
unexportedSliceField []string
|
||||
ExportedMapField map[string]string
|
||||
unexportedMapField map[string]string
|
||||
}
|
||||
|
||||
type supportedMockStruct struct {
|
||||
ExportedField string
|
||||
ExportedSliceField []string
|
||||
ExportedMapField map[string]string
|
||||
}
|
||||
|
||||
func TestDeepCopy(t *testing.T) {
|
||||
t.Run("struct", func(t *testing.T) {
|
||||
t.Run("supported", func(t *testing.T) {
|
||||
srcStruct := supportedMockStruct{
|
||||
ExportedField: "exported field",
|
||||
ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"},
|
||||
ExportedMapField: map[string]string{
|
||||
"key1": "exported map elem",
|
||||
"key2": "exported map elem",
|
||||
},
|
||||
}
|
||||
dstStruct := supportedMockStruct{}
|
||||
|
||||
err := deepCopy(srcStruct, &dstStruct)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("deepCopy returned an unexpected error %+v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(srcStruct, dstStruct) {
|
||||
t.Errorf("deepCopy failed to copy structure: got %+v, want %+v", dstStruct, srcStruct)
|
||||
}
|
||||
})
|
||||
t.Run("unsupported", func(t *testing.T) {
|
||||
srcStruct := unsupportedMockStruct{
|
||||
ExportedField: "exported field",
|
||||
unexportedField: "unexported field",
|
||||
ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"},
|
||||
unexportedSliceField: []string{"1st elem of an unexported slice field", "2nd elem of an unexported slice field"},
|
||||
ExportedMapField: map[string]string{
|
||||
"key1": "exported map elem",
|
||||
"key2": "exported map elem",
|
||||
},
|
||||
unexportedMapField: map[string]string{
|
||||
"key1": "unexported map elem",
|
||||
"key2": "unexported map elem",
|
||||
},
|
||||
}
|
||||
dstStruct := unsupportedMockStruct{}
|
||||
|
||||
err := deepCopy(srcStruct, &dstStruct)
|
||||
|
||||
if err == nil {
|
||||
t.Error("deepCopy was expected to fail copying an structure with unexported fields")
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("map", func(t *testing.T) {
|
||||
t.Run("map[string]string", func(t *testing.T) {
|
||||
srcMap := map[string]string{
|
||||
"key1": "value1",
|
||||
"key2": "value2",
|
||||
}
|
||||
dstMap := make(map[string]string)
|
||||
|
||||
err := deepCopy(srcMap, &dstMap)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("deepCopy returned an unexpected error %+v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(srcMap, dstMap) {
|
||||
t.Errorf("deepCopy failed to copy map: got %+v, want %+v", dstMap, srcMap)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("map[string]struct", func(t *testing.T) {
|
||||
srcMap := map[string]supportedMockStruct{
|
||||
"key1": {
|
||||
ExportedField: "exported field",
|
||||
ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"},
|
||||
ExportedMapField: map[string]string{
|
||||
"key1": "exported map elem",
|
||||
"key2": "exported map elem",
|
||||
},
|
||||
},
|
||||
"key2": {
|
||||
ExportedField: "exported field",
|
||||
ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"},
|
||||
ExportedMapField: map[string]string{
|
||||
"key1": "exported map elem",
|
||||
"key2": "exported map elem",
|
||||
},
|
||||
},
|
||||
}
|
||||
dstMap := make(map[string]supportedMockStruct)
|
||||
|
||||
err := deepCopy(srcMap, &dstMap)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("deepCopy returned an unexpected error %+v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(srcMap, dstMap) {
|
||||
t.Errorf("deepCopy failed to copy map: got %+v, want %+v", dstMap, srcMap)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("slice", func(t *testing.T) {
|
||||
t.Run("[]string", func(t *testing.T) {
|
||||
srcSlice := []string{"A", "B", "C"}
|
||||
dstSlice := make([]string, len(srcSlice))
|
||||
|
||||
err := deepCopy(srcSlice, &dstSlice)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("deepCopy returned an unexpected error %+v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(srcSlice, dstSlice) {
|
||||
t.Errorf("deepCopy failed to copy slice: got %+v, want %+v", dstSlice, srcSlice)
|
||||
}
|
||||
})
|
||||
t.Run("[]struct", func(t *testing.T) {
|
||||
srcSlice := []supportedMockStruct{
|
||||
{
|
||||
ExportedField: "exported field",
|
||||
ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"},
|
||||
ExportedMapField: map[string]string{
|
||||
"key1": "exported map elem",
|
||||
"key2": "exported map elem",
|
||||
},
|
||||
}, {
|
||||
ExportedField: "exported field",
|
||||
ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"},
|
||||
ExportedMapField: map[string]string{
|
||||
"key1": "exported map elem",
|
||||
"key2": "exported map elem",
|
||||
},
|
||||
}, {
|
||||
ExportedField: "exported field",
|
||||
ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"},
|
||||
ExportedMapField: map[string]string{
|
||||
"key1": "exported map elem",
|
||||
"key2": "exported map elem",
|
||||
},
|
||||
},
|
||||
}
|
||||
dstSlice := make([]supportedMockStruct, len(srcSlice))
|
||||
|
||||
err := deepCopy(srcSlice, &dstSlice)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("deepCopy returned an unexpected error %+v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(srcSlice, dstSlice) {
|
||||
t.Errorf("deepCopy failed to copy slice: got %+v, want %+v", dstSlice, srcSlice)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("pointer", func(t *testing.T) {
|
||||
srcStruct := &supportedMockStruct{
|
||||
ExportedField: "exported field",
|
||||
ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"},
|
||||
ExportedMapField: map[string]string{
|
||||
"key1": "exported map elem",
|
||||
"key2": "exported map elem",
|
||||
},
|
||||
}
|
||||
dstStruct := &supportedMockStruct{}
|
||||
|
||||
err := deepCopy(srcStruct, dstStruct)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("deepCopy returned an unexpected error %+v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(srcStruct, dstStruct) {
|
||||
t.Errorf("deepCopy failed to copy structure: got %+v, want %+v", dstStruct, srcStruct)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mismatched", func(t *testing.T) {
|
||||
src := "a string"
|
||||
dst := 123
|
||||
|
||||
err := deepCopy(src, &dst)
|
||||
|
||||
if err == nil {
|
||||
t.Error("deepCopy did not return an error when provided mismatched types")
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -2,31 +2,46 @@ package callbacks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"gorm.io/gorm/schema"
|
||||
"gorm.io/gorm/utils"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Query(db *gorm.DB) {
|
||||
if db.Error == nil {
|
||||
BuildQuerySQL(db)
|
||||
|
||||
if !db.DryRun && db.Error == nil {
|
||||
rows, err := db.Statement.ConnPool.QueryContext(db.Statement.Context, db.Statement.SQL.String(), db.Statement.Vars...)
|
||||
if err != nil {
|
||||
db.AddError(err)
|
||||
return
|
||||
var _query = func(db *gorm.DB) {
|
||||
time.Sleep(5 * time.Second)
|
||||
if !db.DryRun && db.Error == nil {
|
||||
rows, err := db.Statement.ConnPool.QueryContext(db.Statement.Context, db.Statement.SQL.String(), db.Statement.Vars...)
|
||||
if err != nil {
|
||||
db.AddError(err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
db.AddError(rows.Close())
|
||||
}()
|
||||
gorm.Scan(rows, db, 0)
|
||||
}
|
||||
defer func() {
|
||||
db.AddError(rows.Close())
|
||||
}()
|
||||
gorm.Scan(rows, db, 0)
|
||||
}
|
||||
|
||||
if db.Config.Ease {
|
||||
eqDb := db.Ease(_query)
|
||||
if db.Statement.Dest != eqDb.Statement.Dest {
|
||||
if err := deepCopy(eqDb.Statement.Dest, db.Statement.Dest); err != nil {
|
||||
db.AddError(err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
_query(db)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
10
ease.go
Normal file
10
ease.go
Normal file
@ -0,0 +1,10 @@
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
type easedTask struct {
|
||||
wg *sync.WaitGroup
|
||||
db *DB
|
||||
}
|
||||
30
gorm.go
30
gorm.go
@ -56,9 +56,12 @@ type Config struct {
|
||||
Dialector
|
||||
// Plugins registered plugins
|
||||
Plugins map[string]Plugin
|
||||
// Ease database load by grouping identical queries
|
||||
Ease bool
|
||||
|
||||
callbacks *callbacks
|
||||
cacheStore *sync.Map
|
||||
easeQueue *sync.Map
|
||||
}
|
||||
|
||||
// Apply update config to new config
|
||||
@ -167,6 +170,10 @@ func Open(dialector Dialector, opts ...Option) (db *DB, err error) {
|
||||
config.cacheStore = &sync.Map{}
|
||||
}
|
||||
|
||||
if config.Ease && config.easeQueue == nil {
|
||||
config.easeQueue = &sync.Map{}
|
||||
}
|
||||
|
||||
db = &DB{Config: config, clone: 1}
|
||||
|
||||
db.callbacks = initializeCallbacks(db)
|
||||
@ -484,3 +491,26 @@ func (db *DB) ToSQL(queryFn func(tx *DB) *DB) string {
|
||||
|
||||
return db.Dialector.Explain(stmt.SQL.String(), stmt.Vars...)
|
||||
}
|
||||
|
||||
func (db *DB) Ease(cb func(*DB)) *DB {
|
||||
hash := db.Statement.SQL.String()
|
||||
|
||||
eq := &easedTask{
|
||||
wg: &sync.WaitGroup{},
|
||||
db: db,
|
||||
}
|
||||
eq.wg.Add(1)
|
||||
|
||||
var runner, ok = db.easeQueue.LoadOrStore(hash, eq)
|
||||
et := runner.(*easedTask)
|
||||
|
||||
if !ok {
|
||||
cb(db)
|
||||
et.wg.Done()
|
||||
db.easeQueue.Delete(hash)
|
||||
return db
|
||||
}
|
||||
|
||||
et.wg.Wait()
|
||||
return et.db
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user