From 186e8a9e14578c63715444d217294065be072805 Mon Sep 17 00:00:00 2001 From: Cr <631807682@qq.com> Date: Tue, 18 Oct 2022 11:58:42 +0800 Subject: [PATCH 1/6] fix: association without pks (#5779) --- callbacks/associations.go | 10 +++++++-- tests/associations_test.go | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/callbacks/associations.go b/callbacks/associations.go index 00e00fcc..9d7c1412 100644 --- a/callbacks/associations.go +++ b/callbacks/associations.go @@ -208,7 +208,10 @@ func SaveAfterAssociations(create bool) func(db *gorm.DB) { cacheKey := utils.ToStringKey(relPrimaryValues...) if len(relPrimaryValues) != len(rel.FieldSchema.PrimaryFields) || !identityMap[cacheKey] { - identityMap[cacheKey] = true + if cacheKey != "" { // has primary fields + identityMap[cacheKey] = true + } + if isPtr { elems = reflect.Append(elems, elem) } else { @@ -294,7 +297,10 @@ func SaveAfterAssociations(create bool) func(db *gorm.DB) { cacheKey := utils.ToStringKey(relPrimaryValues...) if len(relPrimaryValues) != len(rel.FieldSchema.PrimaryFields) || !identityMap[cacheKey] { - identityMap[cacheKey] = true + if cacheKey != "" { // has primary fields + identityMap[cacheKey] = true + } + distinctElems = reflect.Append(distinctElems, elem) } diff --git a/tests/associations_test.go b/tests/associations_test.go index 42b32afc..4c9076da 100644 --- a/tests/associations_test.go +++ b/tests/associations_test.go @@ -348,3 +348,45 @@ func TestAssociationEmptyQueryClause(t *testing.T) { AssertEqual(t, len(orgs), 0) } } + +type AssociationEmptyUser struct { + ID uint + Name string + Pets []AssociationEmptyPet +} + +type AssociationEmptyPet struct { + AssociationEmptyUserID *uint `gorm:"uniqueIndex:uniq_user_id_name"` + Name string `gorm:"uniqueIndex:uniq_user_id_name;size:256"` +} + +func TestAssociationEmptyPrimaryKey(t *testing.T) { + if DB.Dialector.Name() != "mysql" { + t.Skip() + } + DB.Migrator().DropTable(&AssociationEmptyUser{}, &AssociationEmptyPet{}) + DB.AutoMigrate(&AssociationEmptyUser{}, &AssociationEmptyPet{}) + + id := uint(100) + user := AssociationEmptyUser{ + ID: id, + Name: "jinzhu", + Pets: []AssociationEmptyPet{ + {AssociationEmptyUserID: &id, Name: "bar"}, + {AssociationEmptyUserID: &id, Name: "foo"}, + }, + } + + err := DB.Session(&gorm.Session{FullSaveAssociations: true}).Create(&user).Error + if err != nil { + t.Fatalf("Failed to create, got error: %v", err) + } + + var result AssociationEmptyUser + err = DB.Preload("Pets").First(&result, &id).Error + if err != nil { + t.Fatalf("Failed to find, got error: %v", err) + } + + AssertEqual(t, result, user) +} From ab5f80a8d81c1955e92224b24dfc9bc8c7d387a0 Mon Sep 17 00:00:00 2001 From: Jinzhu Date: Tue, 18 Oct 2022 15:44:47 +0800 Subject: [PATCH 2/6] Save as NULL for nil object serialized into json --- schema/serializer.go | 3 +++ tests/go.mod | 4 ++-- tests/serializer_test.go | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/schema/serializer.go b/schema/serializer.go index 00a4f85f..fef39d9b 100644 --- a/schema/serializer.go +++ b/schema/serializer.go @@ -100,6 +100,9 @@ func (JSONSerializer) Scan(ctx context.Context, field *Field, dst reflect.Value, // Value implements serializer interface func (JSONSerializer) Value(ctx context.Context, field *Field, dst reflect.Value, fieldValue interface{}) (interface{}, error) { result, err := json.Marshal(fieldValue) + if string(result) == "null" { + return nil, err + } return string(result), err } diff --git a/tests/go.mod b/tests/go.mod index 2fef9d97..9c87ca34 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -7,10 +7,10 @@ require ( github.com/jinzhu/now v1.1.5 github.com/lib/pq v1.10.7 golang.org/x/crypto v0.0.0-20221012134737-56aed061732a // indirect - golang.org/x/text v0.3.8 // indirect + golang.org/x/text v0.4.0 // indirect gorm.io/driver/mysql v1.4.3 gorm.io/driver/postgres v1.4.4 - gorm.io/driver/sqlite v1.4.2 + gorm.io/driver/sqlite v1.4.3 gorm.io/driver/sqlserver v1.4.1 gorm.io/gorm v1.24.0 ) diff --git a/tests/serializer_test.go b/tests/serializer_test.go index 946536bf..17bfefe2 100644 --- a/tests/serializer_test.go +++ b/tests/serializer_test.go @@ -18,6 +18,7 @@ type SerializerStruct struct { gorm.Model Name []byte `gorm:"json"` Roles Roles `gorm:"serializer:json"` + Roles2 *Roles `gorm:"serializer:json"` Contracts map[string]interface{} `gorm:"serializer:json"` JobInfo Job `gorm:"type:bytes;serializer:gob"` CreatedTime int64 `gorm:"serializer:unixtime;type:time"` // store time in db, use int as field type @@ -108,7 +109,7 @@ func TestSerializer(t *testing.T) { } var result SerializerStruct - if err := DB.First(&result, data.ID).Error; err != nil { + if err := DB.Where("roles2 IS NULL").First(&result, data.ID).Error; err != nil { t.Fatalf("failed to query data, got error %v", err) } From a0f4d3f7d207b2103b5f91e9758b1ac6a94056ba Mon Sep 17 00:00:00 2001 From: Jinzhu Date: Tue, 18 Oct 2022 16:25:39 +0800 Subject: [PATCH 3/6] Save as empty string for not nullable nil field serialized into json --- schema/serializer.go | 3 +++ tests/serializer_test.go | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/schema/serializer.go b/schema/serializer.go index fef39d9b..9a6aa4fc 100644 --- a/schema/serializer.go +++ b/schema/serializer.go @@ -101,6 +101,9 @@ func (JSONSerializer) Scan(ctx context.Context, field *Field, dst reflect.Value, func (JSONSerializer) Value(ctx context.Context, field *Field, dst reflect.Value, fieldValue interface{}) (interface{}, error) { result, err := json.Marshal(fieldValue) if string(result) == "null" { + if field.TagSettings["NOT NULL"] != "" { + return "", nil + } return nil, err } return string(result), err diff --git a/tests/serializer_test.go b/tests/serializer_test.go index 17bfefe2..a040a4db 100644 --- a/tests/serializer_test.go +++ b/tests/serializer_test.go @@ -19,6 +19,7 @@ type SerializerStruct struct { Name []byte `gorm:"json"` Roles Roles `gorm:"serializer:json"` Roles2 *Roles `gorm:"serializer:json"` + Roles3 *Roles `gorm:"serializer:json;not null"` Contracts map[string]interface{} `gorm:"serializer:json"` JobInfo Job `gorm:"type:bytes;serializer:gob"` CreatedTime int64 `gorm:"serializer:unixtime;type:time"` // store time in db, use int as field type @@ -109,7 +110,7 @@ func TestSerializer(t *testing.T) { } var result SerializerStruct - if err := DB.Where("roles2 IS NULL").First(&result, data.ID).Error; err != nil { + if err := DB.Where("roles2 IS NULL AND roles3 = ?", "").First(&result, data.ID).Error; err != nil { t.Fatalf("failed to query data, got error %v", err) } From 62593cfad03ebf1e6cae30bac010655b4a28ff67 Mon Sep 17 00:00:00 2001 From: viatoriche / Maxim Panfilov Date: Tue, 18 Oct 2022 17:28:06 +0800 Subject: [PATCH 4/6] add test: TestAutoMigrateInt8PG: shouldn't execute ALTER COLUMN TYPE smallint, close #5762 --- migrator/migrator.go | 55 +++++++++++++++++++++---------------------- tests/migrate_test.go | 40 +++++++++++++++++++++++++++++++ tests/tracer_test.go | 34 ++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 28 deletions(-) create mode 100644 tests/tracer_test.go diff --git a/migrator/migrator.go b/migrator/migrator.go index 29c0c00c..9f8e3db8 100644 --- a/migrator/migrator.go +++ b/migrator/migrator.go @@ -406,17 +406,14 @@ func (m Migrator) MigrateColumn(value interface{}, field *schema.Field, columnTy fullDataType := strings.TrimSpace(strings.ToLower(m.DB.Migrator().FullDataTypeOf(field).SQL)) realDataType := strings.ToLower(columnType.DatabaseTypeName()) - alterColumn := false + var ( + alterColumn, isSameType bool + ) if !field.PrimaryKey { // check type - var isSameType bool - if strings.HasPrefix(fullDataType, realDataType) { - isSameType = true - } - - // check type aliases - if !isSameType { + if !strings.HasPrefix(fullDataType, realDataType) { + // check type aliases aliases := m.DB.Migrator().GetTypeAliases(realDataType) for _, alias := range aliases { if strings.HasPrefix(fullDataType, alias) { @@ -424,32 +421,34 @@ func (m Migrator) MigrateColumn(value interface{}, field *schema.Field, columnTy break } } - } - if !isSameType { - alterColumn = true - } - } - - // check size - if length, ok := columnType.Length(); length != int64(field.Size) { - if length > 0 && field.Size > 0 { - alterColumn = true - } else { - // has size in data type and not equal - // Since the following code is frequently called in the for loop, reg optimization is needed here - matches2 := regFullDataType.FindAllStringSubmatch(fullDataType, -1) - if !field.PrimaryKey && - (len(matches2) == 1 && matches2[0][1] != fmt.Sprint(length) && ok) { + if !isSameType { alterColumn = true } } } - // check precision - if precision, _, ok := columnType.DecimalSize(); ok && int64(field.Precision) != precision { - if regexp.MustCompile(fmt.Sprintf("[^0-9]%d[^0-9]", field.Precision)).MatchString(m.DataTypeOf(field)) { - alterColumn = true + if !isSameType { + // check size + if length, ok := columnType.Length(); length != int64(field.Size) { + if length > 0 && field.Size > 0 { + alterColumn = true + } else { + // has size in data type and not equal + // Since the following code is frequently called in the for loop, reg optimization is needed here + matches2 := regFullDataType.FindAllStringSubmatch(fullDataType, -1) + if !field.PrimaryKey && + (len(matches2) == 1 && matches2[0][1] != fmt.Sprint(length) && ok) { + alterColumn = true + } + } + } + + // check precision + if precision, _, ok := columnType.DecimalSize(); ok && int64(field.Precision) != precision { + if regexp.MustCompile(fmt.Sprintf("[^0-9]%d[^0-9]", field.Precision)).MatchString(m.DataTypeOf(field)) { + alterColumn = true + } } } diff --git a/tests/migrate_test.go b/tests/migrate_test.go index b918b4b5..8718aa57 100644 --- a/tests/migrate_test.go +++ b/tests/migrate_test.go @@ -1,6 +1,7 @@ package tests_test import ( + "context" "fmt" "math/rand" "reflect" @@ -9,6 +10,7 @@ import ( "time" "gorm.io/driver/postgres" + "gorm.io/gorm" "gorm.io/gorm/schema" . "gorm.io/gorm/utils/tests" @@ -72,6 +74,44 @@ func TestMigrate(t *testing.T) { t.Fatalf("Failed to find index for many2many for %v %v", indexes[0], indexes[1]) } } + +} + +func TestAutoMigrateInt8PG(t *testing.T) { + if DB.Dialector.Name() != "postgres" { + return + } + + type Smallint int8 + + type MigrateInt struct { + Int8 Smallint + } + + tracer := Tracer{ + Logger: DB.Config.Logger, + Test: func(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) { + sql, _ := fc() + if strings.HasPrefix(sql, "ALTER TABLE \"migrate_ints\" ALTER COLUMN \"int8\" TYPE smallint") { + t.Fatalf("shouldn't execute ALTER COLUMN TYPE if such type is already existed in DB schema: sql: %s", sql) + } + }, + } + + DB.Migrator().DropTable(&MigrateInt{}) + + // The first AutoMigrate to make table with field with correct type + if err := DB.AutoMigrate(&MigrateInt{}); err != nil { + t.Fatalf("Failed to auto migrate: error: %v", err) + } + + // make new session to set custom logger tracer + session := DB.Session(&gorm.Session{Logger: tracer}) + + // The second AutoMigrate to catch an error + if err := session.AutoMigrate(&MigrateInt{}); err != nil { + t.Fatalf("Failed to auto migrate: error: %v", err) + } } func TestAutoMigrateSelfReferential(t *testing.T) { diff --git a/tests/tracer_test.go b/tests/tracer_test.go new file mode 100644 index 00000000..3e9a4052 --- /dev/null +++ b/tests/tracer_test.go @@ -0,0 +1,34 @@ +package tests_test + +import ( + "context" + "time" + + "gorm.io/gorm/logger" +) + +type Tracer struct { + Logger logger.Interface + Test func(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) +} + +func (S Tracer) LogMode(level logger.LogLevel) logger.Interface { + return S.Logger.LogMode(level) +} + +func (S Tracer) Info(ctx context.Context, s string, i ...interface{}) { + S.Logger.Info(ctx, s, i...) +} + +func (S Tracer) Warn(ctx context.Context, s string, i ...interface{}) { + S.Logger.Warn(ctx, s, i...) +} + +func (S Tracer) Error(ctx context.Context, s string, i ...interface{}) { + S.Logger.Error(ctx, s, i...) +} + +func (S Tracer) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) { + S.Logger.Trace(ctx, begin, fc, err) + S.Test(ctx, begin, fc, err) +} From 3f20a543fad5f57016ef7a6c342536b0fcce6016 Mon Sep 17 00:00:00 2001 From: Jinzhu Date: Tue, 18 Oct 2022 18:01:55 +0800 Subject: [PATCH 5/6] Support use clause.Interface as query params --- statement.go | 4 ++++ tests/sql_builder_test.go | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/statement.go b/statement.go index cc26fe37..d05d299e 100644 --- a/statement.go +++ b/statement.go @@ -179,6 +179,10 @@ func (stmt *Statement) AddVar(writer clause.Writer, vars ...interface{}) { } else { stmt.AddVar(writer, v.GormValue(stmt.Context, stmt.DB)) } + case clause.Interface: + c := clause.Clause{Name: v.Name()} + v.MergeClause(&c) + c.Build(stmt) case clause.Expression: v.Build(stmt) case driver.Valuer: diff --git a/tests/sql_builder_test.go b/tests/sql_builder_test.go index b10142fa..0fbd6118 100644 --- a/tests/sql_builder_test.go +++ b/tests/sql_builder_test.go @@ -445,6 +445,14 @@ func TestToSQL(t *testing.T) { if DB.Statement.DryRun || DB.DryRun { t.Fatal("Failed expect DB.DryRun and DB.Statement.ToSQL to be false") } + + // UpdateColumns + sql = DB.ToSQL(func(tx *gorm.DB) *gorm.DB { + return tx.Raw("SELECT * FROM users ?", clause.OrderBy{ + Columns: []clause.OrderByColumn{{Column: clause.Column{Name: "id", Raw: true}, Desc: true}}, + }) + }) + assertEqualSQL(t, `SELECT * FROM users ORDER BY id DESC`, sql) } // assertEqualSQL for assert that the sql is equal, this method will ignore quote, and dialect specials. From 5dd2bb482755f5e8eb5ecaff39e675fb62f19a20 Mon Sep 17 00:00:00 2001 From: Cr <631807682@qq.com> Date: Wed, 19 Oct 2022 14:46:59 +0800 Subject: [PATCH 6/6] feat(PreparedStmtDB): support reset (#5782) * feat(PreparedStmtDB): support reset * fix: close all stmt * test: fix test * fix: delete one by one --- prepare_stmt.go | 12 ++++++++++++ tests/prepared_stmt_test.go | 28 +++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/prepare_stmt.go b/prepare_stmt.go index 3934bb97..7591e533 100644 --- a/prepare_stmt.go +++ b/prepare_stmt.go @@ -44,6 +44,18 @@ func (db *PreparedStmtDB) Close() { } } +func (db *PreparedStmtDB) Reset() { + db.Mux.Lock() + defer db.Mux.Unlock() + for query, stmt := range db.Stmts { + delete(db.Stmts, query) + go stmt.Close() + } + + db.PreparedSQL = make([]string, 0, 100) + db.Stmts = map[string](*Stmt){} +} + func (db *PreparedStmtDB) prepare(ctx context.Context, conn ConnPool, isTransaction bool, query string) (Stmt, error) { db.Mux.RLock() if stmt, ok := db.Stmts[query]; ok && (!stmt.Transaction || isTransaction) { diff --git a/tests/prepared_stmt_test.go b/tests/prepared_stmt_test.go index c7f251f2..64baa01b 100644 --- a/tests/prepared_stmt_test.go +++ b/tests/prepared_stmt_test.go @@ -2,8 +2,8 @@ package tests_test import ( "context" - "sync" "errors" + "sync" "testing" "time" @@ -168,3 +168,29 @@ func TestPreparedStmtInTransaction(t *testing.T) { t.Errorf("Failed, got error: %v", err) } } + +func TestPreparedStmtReset(t *testing.T) { + tx := DB.Session(&gorm.Session{PrepareStmt: true}) + + user := *GetUser("prepared_stmt_reset", Config{}) + tx = tx.Create(&user) + + pdb, ok := tx.ConnPool.(*gorm.PreparedStmtDB) + if !ok { + t.Fatalf("should assign PreparedStatement Manager back to database when using PrepareStmt mode") + } + + pdb.Mux.Lock() + if len(pdb.Stmts) == 0 { + pdb.Mux.Unlock() + t.Fatalf("prepared stmt can not be empty") + } + pdb.Mux.Unlock() + + pdb.Reset() + pdb.Mux.Lock() + defer pdb.Mux.Unlock() + if len(pdb.Stmts) != 0 { + t.Fatalf("prepared stmt should be empty") + } +}