add json serialization+deserialization helpers

.gitignore

@@ -3,3 +3,4 @@ go.work.sum
 go.work
 muck/
 /build/
+/test-logs/

.idea/dataSources.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="testbed_i_think@localhost" uuid="ba7bd11e-a526-49f4-ab27-28ba42665ce5">
+      <driver-ref>postgresql</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.postgresql.Driver</jdbc-driver>
+      <jdbc-url>jdbc:postgresql://localhost:5432/testbed_i_think</jdbc-url>
+      <jdbc-additional-properties>
+        <property name="JdbcLog.Enabled" value="true" />
+      </jdbc-additional-properties>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
+  </component>
+</project>

create_test.go

@@ -0,0 +1,67 @@
+package orm
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func testCreate1(t assert.TestingT, e *Engine) {
+	u := author(t)
+	err := e.Model(&user{}).Save(&u)
+	assert.Nil(t, err)
+	assert.NotZero(t, u.Favs.ID)
+}
+
+func testCreate2(t assert.TestingT, e *Engine) {
+	u := friend(t)
+	err := e.Model(&user{}).Save(&u)
+	assert.Nil(t, err)
+	assert.NotZero(t, u.Favs.ID)
+}
+
+func testCreate3(t assert.TestingT, e *Engine) {
+	insertBands(t, e)
+}
+
+func testCreate4(t assert.TestingT, e *Engine) {
+	u := author(t)
+	err := e.Model(&user{}).Save(&u)
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+	assert.Nil(t, err)
+	insertBands(t, e)
+	storyBase(e, t, u)
+}
+
+func TestCreate1(t *testing.T) {
+	e := initTest(t)
+	testCreate1(t, e)
+	e.Disconnect()
+}
+
+func TestCreate2(t *testing.T) {
+	e := initTest(t)
+	testCreate2(t, e)
+	e.Disconnect()
+}
+
+func TestCreate3(t *testing.T) {
+	e := initTest(t)
+	testCreate3(t, e)
+	e.Disconnect()
+}
+
+func TestCreate4(t *testing.T) {
+	e := initTest(t)
+	testCreate4(t, e)
+	e.Disconnect()
+}
+
+func BenchmarkCreate(b *testing.B) {
+	b.Run("Create-1", bench(testCreate1))
+	b.Run("Create-2", bench(testCreate2))
+	b.Run("Create-3", bench(testCreate3))
+	b.Run("Create-4", bench(testCreate4))
+}

delete_test.go

@@ -0,0 +1 @@
+package orm

diamond.go

@@ -0,0 +1,201 @@
+package orm
+
+import (
+	"context"
+	"fmt"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"io"
+	"log/slog"
+	"os"
+	"rockfic.com/orm/internal/logging"
+	"time"
+)
+
+// LevelQuery enables logging of SQL queries if passed to Config.LogLevel
+const LevelQuery = slog.Level(-6)
+const defaultKey = "default"
+
+type Config struct {
+	DryRun   bool       // when true, queries will not run on the underlying database
+	LogLevel slog.Level // controls the level of information logged; defaults to slog.LevelInfo if not set
+	LogTo    io.Writer  // where to write log output to; defaults to os.Stdout
+}
+
+type Engine struct {
+	modelMap *internalModelMap
+	conn     *pgxpool.Pool
+	m2mSeen  map[string]bool
+	dryRun   bool
+	pgCfg    *pgxpool.Config
+	ctx      context.Context
+	logger   *slog.Logger
+	cfg      *Config
+	levelVar *slog.LevelVar
+	connStr  string
+}
+
+// Models - parse and register one or more types as persistable models
+func (e *Engine) Models(v ...any) {
+	emm := makeModelMap(v...)
+	for k := range emm.Map {
+		if _, ok := e.modelMap.Map[k]; !ok {
+			e.modelMap.Mux.Lock()
+			e.modelMap.Map[k] = emm.Map[k]
+			e.modelMap.Mux.Unlock()
+		}
+	}
+}
+
+// Model - createes a Query and sets its model to
+// the one corresponding to the type of `val`
+func (e *Engine) Model(val any) *Query {
+	qq := &Query{
+		engine:         e,
+		ctx:            context.Background(),
+		wheres:         make(map[string][]any),
+		orders:         make([]string, 0),
+		populationTree: make(map[string]any),
+		joins:          make([]string, 0),
+	}
+	return qq.setModel(val)
+}
+
+// QueryRaw - wrapper for the Query method of pgxpool.Pool
+func (e *Engine) QueryRaw(sql string, args ...any) (pgx.Rows, error) {
+	return e.conn.Query(e.ctx, sql, args...)
+}
+
+// Migrate - non-destructive; run migrations to update the underlying schema, WITHOUT dropping tables beforehand
+func (e *Engine) Migrate() error {
+	failedMigrations := make(map[string]*Model)
+	var err error
+	for mk, m := range e.modelMap.Map {
+		err = m.migrate(e)
+		if err != nil {
+			failedMigrations[mk] = m
+		}
+	}
+
+	for len(failedMigrations) > 0 {
+		e.m2mSeen = make(map[string]bool)
+		for mk, m := range failedMigrations {
+			err = m.migrate(e)
+			if err == nil {
+				delete(failedMigrations, mk)
+			}
+		}
+	}
+	return err
+}
+
+// MigrateDropping - destructive migration; DROP the necessary tables if they exist,
+// then recreate them to match your models' schema
+func (e *Engine) MigrateDropping() error {
+	for _, m := range e.modelMap.Map {
+		sql := fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE;", m.TableName)
+		if _, err := e.conn.Exec(e.ctx, sql); err != nil {
+			return err
+		}
+		for _, r := range m.Relationships {
+			if r.m2mIsh() || r.Type == ManyToMany {
+				jsql := fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE;", r.ComputeJoinTable())
+				if _, err := e.conn.Exec(e.ctx, jsql); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return e.Migrate()
+}
+
+func (e *Engine) logQuery(msg, sql string, args []any) {
+	e.logger.Log(e.ctx, LevelQuery, msg, "sql", sql, "args", logTrunc(200, args))
+}
+
+func (e *Engine) logSql(msg, sql string) {
+	e.logger.Log(e.ctx, LevelQuery, msg, "sql", sql)
+}
+
+// Disconnect - closes and disposes of this Engine's connection pool.
+func (e *Engine) Disconnect() {
+	e.conn.Close()
+	if asFile, ok := e.cfg.LogTo.(*os.File); ok {
+		_ = asFile.Close()
+	}
+}
+
+// Open - creates a new connection according to `connString`
+// and returns a brand new Engine to run FUCK operations on.
+func Open(connString string, cfg *Config) (*Engine, error) {
+	if cfg == nil {
+		cfg = &Config{
+			LogLevel: slog.LevelInfo,
+			LogTo:    os.Stdout,
+			DryRun:   connString == "",
+		}
+	} else {
+		if cfg.LogTo == nil {
+			cfg.LogTo = os.Stdout
+		}
+	}
+	e := &Engine{
+		modelMap: &internalModelMap{
+			Map: make(map[string]*Model),
+		},
+		m2mSeen:  make(map[string]bool),
+		dryRun:   connString == "",
+		ctx:      context.Background(),
+		levelVar: new(slog.LevelVar),
+		cfg:      cfg,
+	}
+	e.levelVar.Set(cfg.LogLevel)
+	replacer := func(groups []string, a slog.Attr) slog.Attr {
+		if a.Key == slog.LevelKey {
+			level := a.Value.Any().(slog.Level)
+			switch level {
+			case LevelQuery:
+				a.Value = slog.StringValue("query")
+			}
+		}
+		return a
+	}
+	e.logger = slog.New(logging.NewFormattedHandler(cfg.LogTo, logging.Options{
+		Level:       e.levelVar,
+		ReplaceAttr: replacer,
+		Format:      "{{.Time}} [{{.Level}}] {{.Message}} | {{ rest }}",
+	}))
+	slog.SetDefault(e.logger)
+	if connString != "" {
+		engines.Mux.Lock()
+		if len(engines.Engines) == 0 || engines.Engines[defaultKey] == nil {
+			engines.Engines[defaultKey] = e
+		} else {
+			engines.Engines[connString] = e
+		}
+		e.connStr = ""
+		engines.Mux.Unlock()
+		var err error
+		e.pgCfg, err = pgxpool.ParseConfig(connString)
+		e.pgCfg.MinConns = 5
+		e.pgCfg.MaxConns = 10
+		e.pgCfg.MaxConnIdleTime = time.Minute * 2
+		e.pgCfg.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
+			oldHandler := conn.Config().OnPgError
+			conn.Config().OnPgError = func(conn *pgconn.PgConn, pgError *pgconn.PgError) bool {
+				e.logger.Error("ERROR ->", "err", pgError.Error())
+				return oldHandler(conn, pgError)
+			}
+			return nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		e.conn, err = pgxpool.NewWithConfig(e.ctx, e.pgCfg)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return e, nil
+}

document.go

@@ -1 +1,15 @@
 package orm
+
+import "time"
+
+// Document - embed this into your structs anonymously to specify
+// model parameters like table name via struct tags
+type Document struct {
+	Created  time.Time `json:"createdAt" tstype:"Date"`
+	Modified time.Time `json:"modifiedAt" tstype:"Date"`
+}
+
+// SaveOptions - unused (for now)
+type SaveOptions struct {
+	SetTimestamps bool
+}

errors.go

@@ -0,0 +1,7 @@
+package orm
+
+import "fmt"
+
+var ErrNoConditionOnDeleteOrUpdate = fmt.Errorf("refusing to delete/update with no conditions specified.\n"+
+	"  (hint: call `.WhereRaw(%s)` or `.WhereRaw(%s)` to do so anyways)",
+	`"true"`, `"1 = 1"`)

field.go

@@ -0,0 +1,159 @@
+package orm
+
+import (
+	"net"
+	"reflect"
+	"time"
+)
+
+// Field - represents a field with a valid SQL type in a Model
+type Field struct {
+	Name          string              // the name of this field as it appears in its Model's type definition
+	ColumnName    string              // this field's snake_cased column name as it appears in database
+	ColumnType    string              // the SQL type of this field's column (bigint, bigserial, text, ...)
+	Type          reflect.Type        // the reflect.Type of the struct field this Field represents
+	Original      reflect.StructField // the raw struct field, as obtained by using reflect.Type.Field or reflect.Type.FieldByName
+	Model         *Model              // the Model this field belongs to
+	Index         int                 // the index at which Original appears in its struct
+	AutoIncrement bool                // whether this field's column is an auto-incrementing column
+	PrimaryKey    bool                // true if this field's column is a primary key
+	Nullable      bool                // true if this field's column can be NULL
+	embeddedFields map[string]*Field // mapping of column names to Field pointers that correspond to the surrounding struct's fields
+}
+
+func (f *Field) isAnonymous() bool {
+	return f.Original.Anonymous
+}
+
+func (f *Field) anonymousColumnNames() []string {
+	cols := make([]string, 0)
+	if !f.isAnonymous() {
+		return cols
+	}
+	for _, ef := range f.embeddedFields {
+		cols = append(cols, ef.ColumnName)
+	}
+	return cols
+}
+
+func defaultColumnValue(ty reflect.Type) any {
+	switch ty.Kind() {
+	case reflect.Int32, reflect.Uint32, reflect.Int, reflect.Uint, reflect.Int64, reflect.Uint64:
+		return 0
+	case reflect.Bool:
+		return false
+	case reflect.String:
+		return "''"
+	case reflect.Float32, reflect.Float64:
+		return 0.0
+	case reflect.Struct:
+		if canConvertTo[time.Time](ty) {
+			return "now()"
+		}
+		if canConvertTo[net.IP](ty) {
+			return "'0.0.0.0'::INET"
+		}
+		if canConvertTo[net.IPNet](ty) {
+			return "'0.0.0.0/0'::CIDR"
+		}
+	case reflect.Slice:
+		return "'{}'"
+	}
+	return "NULL"
+}
+
+func columnType(ty reflect.Type, isPk, isAutoInc bool) string {
+	it := ty
+	switch it.Kind() {
+	case reflect.Ptr:
+		for it.Kind() == reflect.Ptr {
+			it = it.Elem()
+		}
+	case reflect.Int32, reflect.Uint32, reflect.Int, reflect.Uint:
+		if isPk || isAutoInc {
+			return "serial"
+		} else {
+			return "int"
+		}
+	case reflect.Int64, reflect.Uint64:
+		if isPk || isAutoInc {
+			return "bigserial"
+		} else {
+			return "bigint"
+		}
+	case reflect.String:
+		return "text"
+	case reflect.Float32:
+		return "float4"
+	case reflect.Float64:
+		return "double precision"
+	case reflect.Bool:
+		return "boolean"
+	case reflect.Struct:
+		if canConvertTo[time.Time](ty) {
+			return "timestamptz"
+		}
+		if canConvertTo[net.IP](ty) {
+			return "inet"
+		}
+		if canConvertTo[net.IPNet](ty) {
+			return "cidr"
+		}
+
+	default:
+		return ""
+	}
+	return ""
+}
+func parseField(f reflect.StructField, minfo *Model, modelMap map[string]*Model, i int) *Field {
+	field := &Field{
+		Name:     f.Name,
+		Original: f,
+		Index:    i,
+	}
+	tags := parseTags(f.Tag.Get("d"))
+	if tags["-"] != "" {
+		return nil
+	}
+	field.PrimaryKey = tags["pk"] != "" || tags["primarykey"] != "" || field.Name == "ID"
+	field.AutoIncrement = tags["autoinc"] != ""
+	field.Nullable = tags["nullable"] != ""
+	field.ColumnType = tags["type"]
+	if field.ColumnType == "" {
+		field.ColumnType = columnType(f.Type, field.PrimaryKey, field.AutoIncrement)
+	}
+	field.ColumnName = tags["column"]
+	if field.ColumnName == "" {
+		field.ColumnName = pascalToSnakeCase(field.Name)
+	}
+	if field.PrimaryKey {
+		minfo.IDField = field.Name
+	}
+	elem := f.Type
+	for elem.Kind() == reflect.Ptr {
+		if !field.Nullable {
+			field.Nullable = true
+		}
+		elem = elem.Elem()
+	}
+	field.Type = elem
+
+	switch elem.Kind() {
+	case reflect.Array, reflect.Slice:
+		elem = elem.Elem()
+		fallthrough
+	case reflect.Struct:
+		if canConvertTo[Document](elem) && f.Anonymous {
+			minfo.TableName = tags["table"]
+			field.embeddedFields = make(map[string]*Field)
+			for j := range elem.NumField() {
+				efield := elem.Field(j)
+				field.embeddedFields[pascalToSnakeCase(efield.Name)] = parseField(efield, minfo, modelMap, j)
+			}
+		} else if field.ColumnType == "" {
+			minfo.Relationships[field.Name] = parseRelationship(f, modelMap, minfo.Type, i, tags)
+		}
+	}
+
+	return field
+}

find_test.go

@@ -0,0 +1,171 @@
+package orm
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func storyBaseLoop(t assert.TestingT, e *Engine, u user, count int, additionalPopulateFields ...string) {
+	if !isTestBench(t) {
+		for range count {
+			storyBase(e, t, u, additionalPopulateFields...)
+		}
+	} else {
+		storyBase(e, t, u, additionalPopulateFields...)
+	}
+}
+
+func storyBase(e *Engine, t assert.TestingT, u user, additionalPopulateFields ...string) *story {
+	s := iti_multi(u)
+	err := e.Model(&story{}).Save(s)
+	assert.Nil(t, err)
+	assert.NotZero(t, s.ID)
+	checkChapters(t, s)
+	var ns story
+	fields := []string{
+		PopulateAll,
+	}
+	fields = append(fields, additionalPopulateFields...)
+	err = e.Model(&story{}).Where("ID = ?", s.ID).Populate(fields...).Find(&ns)
+	assert.Nil(t, err)
+	assert.NotZero(t, ns.ID)
+	assert.NotZero(t, ns.Author.ID)
+	assert.NotZero(t, ns.Author.Username)
+	assert.Equal(t, len(s.Chapters), len(ns.Chapters))
+	checkChapters(t, &ns)
+	return &ns
+}
+
+func testFind1(t assert.TestingT, e *Engine) {
+	u := author(t)
+	err := e.Model(&user{}).Save(&u)
+	assert.Nil(t, err)
+	var nu user
+	err = e.Model(&user{}).Where("ID = ?", u.ID).Find(&nu)
+	assert.Nil(t, err)
+	assert.NotZero(t, nu.ID)
+	assert.NotZero(t, nu.Username)
+}
+
+func testJoin1(t assert.TestingT, e *Engine) {
+	u := author(t)
+	err := e.Model(&user{}).Save(&u)
+	assert.Nil(t, err)
+	insertBands(t, e)
+	storyBaseLoop(t, e, u, 7)
+	var withBodom []story
+	err = e.Model(&story{}).
+		Join("Chapters.Bands").
+		In("Chapters.Bands.ID", bodom.ID).
+		Find(&withBodom)
+	assert.Nil(t, err)
+	assert.NotEmpty(t, withBodom)
+}
+
+func testIn(t assert.TestingT, e *Engine) {
+	u := author(t)
+	err := e.Model(&user{}).Save(&u)
+	assert.Nil(t, err)
+	insertBands(t, e)
+	storyBaseLoop(t, e, u, 10)
+	var threes []story
+	err = e.Model(&story{}).
+		In("ID", 1, 2, 3).
+		Find(&threes)
+	assert.Nil(t, err)
+	assert.GreaterOrEqual(t, len(threes), 1)
+}
+
+func testPopulateAll(t assert.TestingT, e *Engine) {
+	insertBands(t, e)
+	u := author(t)
+	err := e.Model(&user{}).Create(&u)
+	assert.Nil(t, err)
+	storyBase(e, t, u)
+}
+
+func testPopulateNested1(t assert.TestingT, e *Engine) {
+	u := author(t)
+	f := friend(t)
+
+	err := e.Model(&user{}).Create(&f)
+	assert.Nil(t, err)
+	assert.NotZero(t, f.ID)
+
+	err = e.Model(&user{}).Create(&u)
+	assert.NotZero(t, u.Favs.ID)
+	assert.Nil(t, err)
+
+	u.Favs.Authors = append(u.Favs.Authors, f)
+	err = e.Model(&user{}).Save(&u)
+	assert.Nil(t, err)
+
+	var nu user
+	err = e.Model(&user{}).
+		Populate(PopulateAll, "Favs.Authors").
+		Where("ID = ?", u.ID).
+		Find(&nu)
+	assert.Nil(t, err)
+	assert.NotZero(t, nu.Favs.ID)
+	assert.NotEmpty(t, nu.Favs.Authors)
+}
+
+func testPopulateNested2(t assert.TestingT, e *Engine) {
+	u := author(t)
+	err := e.Model(&user{}).Create(&u)
+	assert.Nil(t, err)
+	insertBands(t, e)
+	ns := storyBase(e, t, u, "Chapters.Bands")
+	for _, c := range ns.Chapters {
+		assert.NotEmpty(t, c.Bands)
+		for _, b := range c.Bands {
+			assert.NotZero(t, b.Name)
+			assert.NotEmpty(t, b.Characters)
+		}
+	}
+}
+
+func TestFind(t *testing.T) {
+	e := initTest(t)
+	testFind1(t, e)
+}
+
+func TestJoin1(t *testing.T) {
+	e := initTest(t)
+	testJoin1(t, e)
+}
+
+func TestIn(t *testing.T) {
+	e := initTest(t)
+	testIn(t, e)
+}
+
+func TestPopulateAll(t *testing.T) {
+	e := initTest(t)
+	testPopulateAll(t, e)
+}
+
+func TestPopulateNested1(t *testing.T) {
+	e := initTest(t)
+	testPopulateNested1(t, e)
+}
+
+func TestPopulateNested2(t *testing.T) {
+	e := initTest(t)
+	testPopulateNested2(t, e)
+}
+
+func BenchmarkFind(b *testing.B) {
+	b.Run("Plain-1", bench(testFind1))
+	b.Run("Join-1", bench(testJoin1))
+	b.Run("In-1", bench(testIn))
+}
+
+func BenchmarkPopulate(b *testing.B) {
+	b.Run("Populate", func(b *testing.B) {
+
+		b.Run("Simple-1", bench(testPopulateAll))
+		b.Run("Populate-Nested-1", bench(testPopulateNested1))
+		b.Run("Populate-Nested-2", bench(testPopulateNested2))
+	})
+}

go.mod

@@ -3,13 +3,30 @@ module rockfic.com/orm
 go 1.24.1
 
 require (
-	github.com/go-loremipsum/loremipsum v1.1.4 // indirect
+	github.com/go-loremipsum/loremipsum v1.1.4
+	github.com/huandu/go-sqlbuilder v1.35.1
+	github.com/jackc/pgx/v5 v5.7.5
+	github.com/jinzhu/now v1.1.5
+	github.com/stretchr/testify v1.10.0
+)
+
+require (
+	github.com/Jeffail/gabs v1.4.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/henvic/pgq v0.0.4 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/pgx/v5 v5.7.5 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/stretchr/testify v1.10.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	golang.org/x/crypto v0.37.0 // indirect
 	golang.org/x/sync v0.13.0 // indirect
 	golang.org/x/text v0.24.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace rockfic.com/orm => C:/rockfic/orm

go.sum

@@ -1,6 +1,23 @@
+github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo=
+github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
+github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/go-loremipsum/loremipsum v1.1.4 h1:RJaJlJwX4y9A2+CMgKIyPcjuFHFKTmaNMhxbL+sI6Vg=
 github.com/go-loremipsum/loremipsum v1.1.4/go.mod h1:whNWskGoefTakPnCu2CO23v5Y7RwiG4LMOEtTDaBeOY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/henvic/pgq v0.0.4 h1:BgLnxofZJSWWs+9VOf19Gr9uBkSVbHWGiu8wix1nsIY=
+github.com/henvic/pgq v0.0.4/go.mod h1:k0FMvOgmQ45MQ3TgCLe8I3+sDKy9lPAiC2m9gg37pVA=
+github.com/huandu/go-assert v1.1.6 h1:oaAfYxq9KNDi9qswn/6aE0EydfxSa+tWZC1KabNitYs=
+github.com/huandu/go-assert v1.1.6/go.mod h1:JuIfbmYG9ykwvuxoJ3V8TB5QP+3+ajIA54Y44TmkMxs=
+github.com/huandu/go-sqlbuilder v1.35.1 h1:znTuAksxq3T1rYfr3nsD4P0brWDY8qNzdZnI6+vtia4=
+github.com/huandu/go-sqlbuilder v1.35.1/go.mod h1:mS0GAtrtW+XL6nM2/gXHRJax2RwSW1TraavWDFAc1JA=
+github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
+github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -9,7 +26,16 @@ github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs=
 github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -22,4 +48,8 @@ golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
 golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/logging/custom_handler.go

@@ -0,0 +1,207 @@
+package logging
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"log/slog"
+	"runtime"
+	"slices"
+	"strings"
+	"sync"
+	"text/template"
+	"time"
+)
+
+type FormattedHandler struct {
+	mu       *sync.Mutex
+	out      io.Writer
+	opts     Options
+	attrs    map[string]slog.Value
+	groups   []string
+	groupLvl int
+}
+
+type Options struct {
+	Level       slog.Leveler
+	Format      string
+	ReplaceAttr func(groups []string, attr slog.Attr) slog.Attr
+}
+type locData struct {
+	FileName string
+	Function string
+	Line     int
+}
+
+func NewFormattedHandler(out io.Writer, options Options) *FormattedHandler {
+	h := &FormattedHandler{
+		opts:   options,
+		out:    out,
+		mu:     &sync.Mutex{},
+		groups: make([]string, 0),
+	}
+	if h.opts.Format == "" {
+		h.opts.Format = "{{.Time}} [{{.Level}}]"
+	}
+	if h.opts.Level == nil {
+		h.opts.Level = slog.LevelInfo
+	}
+	return h
+}
+
+func (f *FormattedHandler) Enabled(ctx context.Context, level slog.Level) bool {
+	return level >= f.opts.Level.Level()
+}
+
+func (f *FormattedHandler) Handle(ctx context.Context, r slog.Record) error {
+	bufp := allocBuf()
+	buf := *bufp
+	defer func() {
+		*bufp = buf
+		freeBuf(bufp)
+	}()
+	rep := f.opts.ReplaceAttr
+	key := slog.LevelKey
+	val := r.Level
+	if rep == nil {
+		r.AddAttrs(slog.String(key, val.String()))
+	} else {
+		nattr := slog.Any(key, val)
+		nattr.Value = rep(f.groups, nattr).Value
+		r.AddAttrs(nattr)
+	}
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	tctx, tmpl := f.newFmtCtx(r)
+	wr := bytes.NewBuffer(buf)
+	parsed, err := tmpl.Parse(f.opts.Format)
+	if err != nil {
+		return err
+	}
+	err = parsed.Execute(wr, tctx)
+	if err != nil {
+		return err
+	}
+
+	wr.WriteByte('\n')
+	_, err = f.out.Write(wr.Bytes())
+	return err
+}
+
+func (f *FormattedHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+	if len(attrs) == 0 {
+		return f
+	}
+	nf := f.clone()
+	bufp := allocBuf()
+	buf := *bufp
+	defer func() {
+		*bufp = buf
+		freeBuf(bufp)
+	}()
+	s := f.newState(bytes.NewBuffer(buf))
+	defer s.free()
+	pos := s.buf.Len()
+	s.startGroups()
+	if !s.appendAttrs(attrs) {
+		s.buf.Truncate(pos)
+	} else {
+		nf.groupLvl = len(nf.groups)
+	}
+	return nf
+}
+
+func (f *FormattedHandler) WithGroup(name string) slog.Handler {
+	if name == "" {
+		return f
+	}
+	f2 := f.clone()
+	f2.groups = append(f2.groups, name)
+	return f2
+}
+
+func (f *FormattedHandler) clone() *FormattedHandler {
+	return &FormattedHandler{
+		opts:     f.opts,
+		groups:   slices.Clip(f.groups),
+		out:      f.out,
+		mu:       f.mu,
+		groupLvl: f.groupLvl,
+	}
+}
+
+type tmplData struct {
+	Level    string
+	Message  string
+	RawTime  time.Time
+	Time     string
+	PC       uintptr
+	Location locData
+	Record   slog.Record
+}
+
+func hasBuiltInKey(a slog.Attr) bool {
+	return a.Key == slog.MessageKey ||
+		a.Key == slog.TimeKey ||
+		a.Key == slog.SourceKey
+}
+
+func (f *FormattedHandler) newFmtCtx(r slog.Record) (ctx *tmplData, tmpl *template.Template) {
+	tmpl = template.New("log")
+	ctx = &tmplData{
+		Message:  r.Message,
+		RawTime:  r.Time,
+		PC:       r.PC,
+		Location: locData{},
+	}
+	if !r.Time.IsZero() {
+		ctx.Time = r.Time.Format(time.RFC3339Nano)
+	}
+	r.Attrs(func(a slog.Attr) bool {
+		if a.Key == slog.LevelKey {
+			str := strings.ToUpper(a.Value.String())
+			if rep := f.opts.ReplaceAttr; rep != nil {
+				str = strings.ToUpper(a.Value.String())
+			}
+			ctx.Level = str
+		}
+		return true
+	})
+	if r.PC != 0 {
+		frames := runtime.CallersFrames([]uintptr{r.PC})
+		frame, _ := frames.Next()
+		ctx.Location.FileName = frame.File
+		ctx.Location.Function = frame.Function
+		ctx.Location.Line = frame.Line
+	}
+	fm := make(map[string]any)
+	fm["rest"] = func() string {
+		bb := new(bytes.Buffer)
+		s := f.newState(bb)
+		defer s.free()
+		s.begin(r)
+		return s.buf.String()
+	}
+	tmpl = tmpl.Funcs(fm)
+	return
+}
+
+var bufPool = sync.Pool{
+	New: func() any {
+		b := make([]byte, 0, 4096)
+		return &b
+	},
+}
+
+func allocBuf() *[]byte {
+	return bufPool.Get().(*[]byte)
+}
+
+func freeBuf(b *[]byte) {
+	const maxBufferSize = 16 << 10
+	if cap(*b) <= maxBufferSize {
+		*b = (*b)[:0]
+		bufPool.Put(b)
+	}
+}

internal/logging/custom_handler_state.go

@@ -0,0 +1,145 @@
+package logging
+
+import (
+	"bytes"
+	"fmt"
+	"log/slog"
+	"strings"
+	"sync"
+	"time"
+)
+
+func (f *FormattedHandler) newState(sb *bytes.Buffer) state {
+	s := state{
+		fh:  f,
+		buf: sb,
+	}
+	if f.opts.ReplaceAttr != nil {
+		s.groups = groupPool.Get().(*[]string)
+		*s.groups = append(*s.groups, f.groups[:f.groupLvl]...)
+	}
+	return s
+}
+
+type state struct {
+	buf    *bytes.Buffer
+	fh     *FormattedHandler
+	groups *[]string
+}
+
+func (s *state) startGroups() {
+	for _, n := range s.fh.groups[s.fh.groupLvl:] {
+		s.startGroup(n)
+	}
+}
+
+func (s *state) startGroup(name string) {
+	s.buf.WriteByte('\n')
+	if s.groups != nil {
+		*s.groups = append(*s.groups, name)
+	}
+}
+
+func (s *state) endGroup() {
+	if s.groups != nil {
+		*s.groups = (*s.groups)[:len(*s.groups)-1]
+	}
+}
+
+func (s *state) appendAttr(a slog.Attr) bool {
+	a.Value = a.Value.Resolve()
+	if rep := s.fh.opts.ReplaceAttr; rep != nil && a.Value.Kind() != slog.KindGroup {
+		var gs []string
+		if s.groups != nil {
+			gs = *s.groups
+		}
+		a = rep(gs, a)
+		a.Value = a.Value.Resolve()
+	}
+	if a.Equal(slog.Attr{}) ||
+		hasBuiltInKey(a) ||
+		a.Key == slog.LevelKey {
+		return false
+	}
+	if a.Value.Kind() == slog.KindGroup {
+		pos := s.buf.Len()
+		attrs := a.Value.Group()
+		if len(attrs) > 0 {
+			if a.Key != "" {
+				s.startGroup(a.Key)
+			}
+			if !s.appendAttrs(attrs) {
+				s.buf.Truncate(pos)
+				return false
+			}
+			if a.Key != "" {
+				s.endGroup()
+			}
+		}
+	} else {
+		s.writeAttr(a)
+	}
+	return true
+}
+
+func (s *state) appendAttrs(as []slog.Attr) bool {
+	nonEmpty := false
+	for _, a := range as {
+		if s.appendAttr(a) {
+			nonEmpty = true
+		}
+	}
+	return nonEmpty
+}
+
+func (s *state) writeAttr(a slog.Attr) {
+	if s.buf.Len() > 0 {
+		s.buf.WriteString(";")
+	}
+	if len(*s.groups) > 0 {
+		s.buf.WriteString(fmt.Sprintf("%*s", len(*s.groups)*2, ""))
+		s.buf.WriteString(strings.Join(*s.groups, "."))
+		s.buf.WriteString(".")
+	}
+	s.buf.WriteString(a.Key)
+	s.buf.WriteString("=")
+	switch a.Value.Kind() {
+	case slog.KindDuration:
+		s.buf.WriteString(a.Value.Duration().String())
+	case slog.KindTime:
+		s.buf.WriteString(a.Value.Time().Format(time.RFC3339Nano))
+	default:
+		s.buf.WriteString(fmt.Sprintf("%+v", a.Value.Any()))
+	}
+}
+
+func (s *state) begin(r slog.Record) {
+	if r.NumAttrs() > 0 {
+		pos := s.buf.Len()
+		s.startGroups()
+		empty := true
+		r.Attrs(func(a slog.Attr) bool {
+			isBuiltIn := hasBuiltInKey(a) || a.Key == slog.LevelKey
+			if !isBuiltIn && s.appendAttr(a) {
+				empty = false
+			}
+			return true
+		})
+		if empty {
+			s.buf.Truncate(pos)
+		}
+	}
+}
+
+func (s *state) free() {
+	if gs := s.groups; gs != nil {
+		*gs = (*gs)[:0]
+		groupPool.Put(gs)
+	}
+	s.buf.Reset()
+}
+
+var groupPool = sync.Pool{New: func() any {
+	s := make([]string, 0, 10)
+	return &s
+}}

internal/logging/custom_handler_test.go

@@ -0,0 +1,34 @@
+package logging
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"testing"
+)
+
+const LevelQ = slog.Level(-6)
+
+func TestDoAFlip(t *testing.T) {
+	t.Name()
+	replacer := func(groups []string, a slog.Attr) slog.Attr {
+		if a.Key == slog.LevelKey {
+			level := a.Value.Any().(slog.Level)
+			switch level {
+			case LevelQ:
+				a.Value = slog.StringValue("q")
+			}
+		}
+		return a
+	}
+	h := NewFormattedHandler(os.Stderr, Options{
+		Format:      "{{.Time}} [{{.Level}}] {{.Message}} | {{ rest }}",
+		Level:       LevelQ,
+		ReplaceAttr: replacer,
+	})
+	logger := slog.New(h)
+	slog.SetDefault(logger)
+
+	logger.Debug("hello", "btfash", true)
+	logger.Log(context.TODO(), LevelQ, "hi")
+}

json.go

@@ -0,0 +1,319 @@
+package orm
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/fatih/structtag"
+	"reflect"
+	"time"
+)
+
+func defaultEngine() *Engine {
+	return engines.Engines[defaultKey]
+}
+
+func anyToModel(input any) *Model {
+	rv := reflect.TypeOf(input)
+	for rv.Kind() == reflect.Ptr ||
+		rv.Kind() == reflect.Interface ||
+		rv.Kind() == reflect.Slice || rv.Kind() == reflect.Pointer {
+		rv = rv.Elem()
+	}
+	maybeEngine := defaultEngine()
+	if maybeEngine == nil {
+		return nil
+	}
+	return maybeEngine.modelMap.Map[rv.Name()]
+}
+
+func JSONSerialize(input any, pretty bool) ([]byte, error) {
+	vp := reflect.ValueOf(input)
+	vt := reflect.TypeOf(input)
+	if vt.Kind() != reflect.Pointer {
+		return nil, fmt.Errorf("Argument must be a pointer or pointer to a slice; got: %v", vt.Kind())
+	}
+	ser, err := innerSerialize(vp)
+	if err != nil {
+		return nil, err
+	}
+	if pretty {
+		return json.MarshalIndent(ser, "", "\t")
+	}
+	return json.Marshal(ser)
+}
+
+func JSONDeserialize(val any, ser []byte) error {
+	var fiv any
+	if err := json.Unmarshal(ser, &fiv); err != nil {
+		return err
+	}
+	vp := reflect.ValueOf(val)
+	if vp.Kind() != reflect.Pointer {
+		return fmt.Errorf("Argument must be a pointer or pointer to a slice; got: %v", vp.Kind())
+	}
+	m := anyToModel(val)
+	if m == nil {
+		return fmt.Errorf("No model found for type '%s'", vp.Type().Name())
+	}
+	maybeEngine := defaultEngine()
+	if maybeEngine == nil {
+		return fmt.Errorf("No engines have been created!?")
+	}
+	fv, err := innerDeserialize(fiv, m, maybeEngine)
+	if err != nil {
+		return err
+	}
+	vp.Elem().Set(fv)
+	return nil
+}
+
+func innerSerialize(v reflect.Value) (ret any, err error) {
+	switch v.Kind() {
+	case reflect.Interface:
+		v = v.Elem()
+		fallthrough
+	case reflect.Pointer:
+		if v.IsNil() {
+			return ret, nil
+		}
+		for v.Kind() == reflect.Ptr {
+			v = v.Elem()
+		}
+		if v.IsZero() {
+			return ret, nil
+		}
+		fallthrough
+	case reflect.Struct:
+		m := anyToModel(v.Interface())
+
+		if m == nil {
+			if canConvertTo[time.Time](v.Type()) {
+				ret = v.Interface().(time.Time).Format(time.RFC3339)
+			} else {
+				var bytes []byte
+				bytes, err = json.Marshal(v.Interface())
+				if err != nil {
+					return nil, err
+				}
+				ser := make(map[string]any)
+				err = json.Unmarshal(bytes, &ser)
+				if err != nil {
+					return nil, err
+				}
+				ret = ser
+			}
+
+		} else {
+			depopulated, depopulatedId := isDepopulated(v, m.IDField)
+			if depopulated {
+				ret = depopulatedId
+			} else {
+				rmap := make(map[string]any)
+				for i := range v.NumField() {
+					fv := v.Field(i)
+					ft := v.Type().Field(i)
+					var tag *structtag.Tags
+					tag, err = structtag.Parse(string(ft.Tag))
+					if err != nil {
+						return nil, err
+					}
+					var jsonTag *structtag.Tag
+					jsonTag, err = tag.Get("json")
+					if err != nil || jsonTag.Name == "-" {
+						continue
+					}
+					if jsonTag.Name == "" {
+						// we are dealing with an inlined/anonymous struct
+						var maybeMap any
+						maybeMap, err = innerSerialize(fv)
+						if amap, ok := maybeMap.(map[string]any); ok {
+							for k, vv := range amap {
+								rmap[k] = vv
+							}
+						}
+					} else {
+						rmap[jsonTag.Name], err = innerSerialize(fv)
+						if err != nil {
+							return nil, err
+						}
+					}
+				}
+				ret = rmap
+			}
+		}
+	case reflect.Slice, reflect.Array:
+		ret0 := make([]any, 0)
+		for i := range v.Len() {
+			var ser any
+			ser, err = innerSerialize(v.Index(i))
+			if err != nil {
+				return nil, err
+			}
+			ret0 = append(ret0, ser)
+		}
+		ret = ret0
+	default:
+		ret = v.Interface()
+	}
+	return ret, nil
+}
+
+func innerDeserialize(input any, m *Model, e *Engine) (nv reflect.Value, err error) {
+	t := m.Type
+	irv := reflect.ValueOf(input)
+	if irv.Kind() == reflect.Slice || irv.Kind() == reflect.Array {
+		nv = reflect.MakeSlice(reflect.SliceOf(t), 0, 0)
+		for i := range irv.Len() {
+			var snv reflect.Value
+			cur := irv.Index(i)
+			snv, err = innerDeserialize(cur.Interface(), m, e)
+			if err != nil {
+				return snv, err
+			}
+			nv = reflect.Append(nv, snv)
+		}
+	} else { // it's a map or primitive value
+		nv = reflect.New(t).Elem()
+		if asMap, ok := input.(map[string]any); ok {
+			for _, f := range m.Fields {
+				if f.Index < 0 {
+					continue
+				}
+				ft := f.Original
+				fv := nv.Field(f.Index)
+				var tags *structtag.Tags
+				var btag *structtag.Tag
+				tags, err = structtag.Parse(string(ft.Tag))
+				if err != nil {
+					return
+				}
+				btag, err = tags.Get("json")
+				if err != nil || btag.Name == "-" {
+					continue
+				}
+				interm := asMap[btag.Name]
+				var tmp any
+				if str, sok := interm.(string); sok {
+					if ttmp, terr := time.Parse(time.RFC3339, str); terr == nil {
+						tmp = ttmp
+					} else {
+						tmp = interm
+					}
+				} else {
+					tmp = interm
+				}
+				switch fv.Kind() {
+				case reflect.Int64, reflect.Int32, reflect.Int, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+					if tmp != nil {
+						fv.Set(reflect.ValueOf(tmp).Convert(ft.Type))
+					}
+				case reflect.Array, reflect.Slice:
+					if interm != nil {
+
+						slic := reflect.ValueOf(interm)
+						fv.Set(handleSliceMaybe(slic, fv.Type().Elem()))
+					}
+				default:
+					if ft.Anonymous {
+						var nfv reflect.Value
+						nfv, err = handleAnon(input, ft.Type)
+						if err != nil {
+							return
+						}
+						fv.Set(nfv)
+					} else {
+						fv.Set(reflect.ValueOf(tmp))
+					}
+				}
+			}
+			for _, r := range m.Relationships {
+				if r.Type == BelongsTo || r.Type == ManyToOne || r.Idx < 0 {
+					continue
+				}
+				ft := r.OriginalField
+				fv := nv.Field(r.Idx)
+				var tags *structtag.Tags
+				var btag *structtag.Tag
+				tags, err = structtag.Parse(string(ft.Tag))
+				if err != nil {
+					return
+				}
+				btag, err = tags.Get("json")
+				if err != nil || btag.Name == "-" {
+					continue
+				}
+				var rv reflect.Value
+				interm := asMap[btag.Name]
+				rv, err = innerDeserialize(interm, r.RelatedModel, e)
+				if err != nil {
+					return reflect.Value{}, err
+				}
+				fv.Set(rv)
+			}
+		} else {
+			iface := nv.Addr().Interface()
+			err = e.Model(iface).Where(fmt.Sprintf("%s = ?", m.IDField), input).Find(iface)
+			if err != nil {
+				return reflect.Value{}, err
+			}
+		}
+	}
+	return
+}
+
+func handleAnon(raw any, rtype reflect.Type) (nv reflect.Value, err error) {
+	nv = reflect.New(rtype).Elem()
+	amap, ok := raw.(map[string]any)
+	if ok {
+		for i := range rtype.NumField() {
+			ft := rtype.Field(i)
+			fv := nv.Field(i)
+			var tags *structtag.Tags
+			var btag *structtag.Tag
+			tags, err = structtag.Parse(string(ft.Tag))
+			if err != nil {
+				return
+			}
+			btag, terr := tags.Get("json")
+			if terr != nil || btag.Name == "-" || !ft.IsExported() {
+				continue
+			}
+			fval := amap[btag.Name]
+			if reflect.TypeOf(fval) == reflect.TypeFor[string]() && ft.Type == reflect.TypeFor[time.Time]() {
+				tt, _ := time.Parse(time.RFC3339, fval.(string))
+				fv.Set(reflect.ValueOf(tt))
+			} else if fval != nil {
+				fv.Set(reflect.ValueOf(fval))
+			}
+		}
+	}
+	return
+}
+func handleSliceMaybe(iv reflect.Value, dstType reflect.Type) reflect.Value {
+	if iv.Kind() != reflect.Slice && iv.Kind() != reflect.Pointer {
+		return iv
+	}
+	dst := reflect.MakeSlice(reflect.SliceOf(dstType), 0, 0)
+	for i := range iv.Len() {
+		//dst.Set(reflect.Append(fv, handleSliceMaybe(iv.Index(i).Elem())))
+		maybeIface := iv.Index(i)
+		if maybeIface.Kind() == reflect.Interface {
+			maybeIface = maybeIface.Elem()
+		}
+		maybeNdest := dstType
+		if maybeNdest.Kind() == reflect.Slice || maybeNdest.Kind() == reflect.Array {
+			maybeNdest = maybeNdest.Elem()
+		}
+		dst = reflect.Append(dst, handleSliceMaybe(maybeIface, maybeNdest))
+	}
+	return dst
+}
+func isDepopulated(v reflect.Value, idField string) (bool, any) {
+	for v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	syn := reflect.New(v.Type()).Elem()
+	syn.FieldByName(idField).Set(v.FieldByName(idField))
+	finalId := v.FieldByName(idField).Interface()
+	return reflect.DeepEqual(v.Interface(), syn.Interface()), finalId
+}

json_test.go

@@ -0,0 +1,74 @@
+package orm
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/Jeffail/gabs"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestJsonSerialize(t *testing.T) {
+	e := initTest(t)
+	defer e.Disconnect()
+	u := author(t)
+	err := e.Model(&user{}).Save(&u)
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+	assert.Nil(t, err)
+	insertBands(t, e)
+	ns := storyBase(e, t, u, "Chapters.Bands")
+	bytes, err := JSONSerialize(ns, true)
+	assert.Nil(t, err)
+	fmt.Println(string(bytes))
+}
+
+func TestJSONDeserialize(t *testing.T) {
+	e := initTest(t)
+	defer e.Disconnect()
+	u := author(t)
+	err := e.Model(&user{}).Save(&u)
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+	assert.Nil(t, err)
+	insertBands(t, e)
+	ns := storyBase(e, t, u, "Chapters.Bands")
+	bytes, err := JSONSerialize(ns, true)
+	assert.Nil(t, err)
+	fmt.Println(string(bytes))
+	msi := make(map[string]any)
+	err = json.Unmarshal(bytes, &msi)
+	assert.Nil(t, err)
+	obj, err := gabs.Consume(msi)
+	assert.Nil(t, err)
+	children, err := obj.S("chapters").Children()
+	assert.Nil(t, err)
+	for _, child := range children {
+		bands := child.S("bands")
+		var bcontainer []*gabs.Container
+		bcontainer, err = bands.Children()
+		assert.Nil(t, err)
+		if err != nil {
+			break
+		}
+		for j := range bcontainer {
+			id := bcontainer[j].S("_id").Data()
+			//obj.S("chapters").Index(i).S("bands").Index
+			_, err = bands.SetIndex(id, j)
+			assert.Nil(t, err)
+			if err != nil {
+				break
+			}
+		}
+	}
+	nbytes := obj.Bytes()
+	var des story
+	err = JSONDeserialize(&des, nbytes)
+	assert.Nil(t, err)
+	for _, c := range des.Chapters {
+		assert.NotNil(t, c.Bands)
+		assert.GreaterOrEqual(t, len(c.Bands), 1)
+	}
+}

model.go

@@ -0,0 +1,66 @@
+package orm
+
+import (
+	"reflect"
+)
+
+// Model - an intermediate representation of a Go struct
+type Model struct {
+	Name               string                   // the name, almost always the name of the underlying Type
+	Type               reflect.Type             // the Go type this model represents
+	Relationships      map[string]*Relationship // a mapping of struct field names to Relationship pointers
+	IDField            string                   // the name of the field containing this model's ID
+	Fields             map[string]*Field        // mapping of struct field names to Field pointers
+	FieldsByColumnName map[string]*Field        // mapping of database column names to Field pointers
+	TableName          string                   // the name of the table where this model's data is stored. defaults to a snake_cased version of the type/struct name if not provided explicitly via tag
+	embeddedIsh        bool                     // INTERNAL - whether this model is: 1) contained in another type and 2) wasn't explicitly passed to the `Models` method (i.e., it can't "exist" on its own)
+}
+
+func (m *Model) addField(field *Field) {
+	field.Model = m
+	m.Fields[field.Name] = field
+	m.FieldsByColumnName[field.ColumnName] = field
+}
+
+const (
+	documentField = "Document"
+	createdField  = "Created"
+	modifiedField = "Modified"
+)
+
+func (m *Model) docField() *Field {
+	return m.Fields[documentField]
+}
+
+func (m *Model) idField() *Field {
+	return m.Fields[m.IDField]
+}
+
+func (m *Model) getPrimaryKey(val reflect.Value) (string, any) {
+	colField := m.Fields[m.IDField]
+	if colField == nil {
+		return "", nil
+	}
+	colName := colField.ColumnName
+	wasPtr := false
+	if val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return "", nil
+		}
+		val = val.Elem()
+		wasPtr = true
+	}
+	if val.IsZero() && wasPtr {
+		return "", nil
+	}
+	idField := val.FieldByName(m.IDField)
+	if idField.IsValid() {
+		return colName, idField.Interface()
+	}
+	return "", nil
+}
+
+func (m *Model) needsPrimaryKey(val reflect.Value) bool {
+	_, pk := m.getPrimaryKey(val)
+	return pk == nil || reflect.ValueOf(pk).IsZero()
+}

model_internals.go

@@ -0,0 +1,105 @@
+package orm
+
+import (
+	"reflect"
+	"strings"
+)
+
+func parseModel(model any) *Model {
+	t := reflect.TypeOf(model)
+	for t.Kind() == reflect.Ptr {
+		t = t.Elem()
+	}
+	minfo := &Model{
+		Name:               t.Name(),
+		Relationships:      make(map[string]*Relationship),
+		Fields:             make(map[string]*Field),
+		FieldsByColumnName: make(map[string]*Field),
+		Type:               t,
+	}
+	for i := range t.NumField() {
+		f := t.Field(i)
+		if !f.IsExported() {
+			continue
+		}
+	}
+	if minfo.TableName == "" {
+		minfo.TableName = pascalToSnakeCase(t.Name())
+	}
+	return minfo
+}
+
+func parseModelFields(model *Model, modelMap map[string]*Model) {
+	t := model.Type
+	for i := range t.NumField() {
+		f := t.Field(i)
+		fi := parseField(f, model, modelMap, i)
+		if fi != nil && (fi.ColumnType != "" || fi.isAnonymous()) {
+			model.addField(fi)
+		}
+	}
+}
+func makeModelMap(models ...any) *internalModelMap {
+	modelMap := &internalModelMap{
+		Map: make(map[string]*Model),
+	}
+	//internalModelMap := make(map[string]*Model)
+	for _, model := range models {
+		minfo := parseModel(model)
+		modelMap.Mux.Lock()
+		modelMap.Map[minfo.Name] = minfo
+		modelMap.Mux.Unlock()
+	}
+	for _, model := range modelMap.Map {
+		modelMap.Mux.Lock()
+		parseModelFields(model, modelMap.Map)
+		modelMap.Mux.Unlock()
+	}
+	tagManyToMany(modelMap)
+	for _, model := range modelMap.Map {
+		for _, ref := range model.Relationships {
+			if ref.Type != ManyToMany && ref.Idx != -1 {
+				modelMap.Mux.Lock()
+				addForeignKeyFields(ref)
+				modelMap.Mux.Unlock()
+			}
+		}
+	}
+	return modelMap
+}
+
+func tagManyToMany(models *internalModelMap) {
+	hasManys := make(map[string]*Relationship)
+	for _, model := range models.Map {
+		for relName := range model.Relationships {
+			hasManys[model.Name+"."+relName] = model.Relationships[relName]
+		}
+	}
+	for _, model := range models.Map {
+		models.Mux.Lock()
+		for relName := range model.Relationships {
+
+			mb := model.Relationships[relName].RelatedModel
+			var name string
+			for n, reltmp := range hasManys {
+				if !strings.HasPrefix(n, mb.Name) || reltmp.Type != HasMany {
+					continue
+				}
+				if reltmp.RelatedType == model.Type {
+					name = reltmp.FieldName
+					break
+				}
+			}
+			if rel2, ok := mb.Relationships[name]; ok {
+				if name < relName &&
+					rel2.Type == HasMany && model.Relationships[relName].Type == HasMany {
+					mb.Relationships[name].Type = ManyToMany
+					mb.Relationships[name].m2mInverse = model.Relationships[relName]
+					model.Relationships[relName].Type = ManyToMany
+					model.Relationships[relName].m2mInverse = mb.Relationships[name]
+				}
+			}
+		}
+		models.Mux.Unlock()
+	}
+}

model_map.go

@@ -0,0 +1,20 @@
+package orm
+
+import (
+	"sync"
+)
+
+type internalModelMap struct {
+	Map map[string]*Model
+	Mux sync.RWMutex
+}
+
+type engineHolder struct {
+	Engines map[string]*Engine
+	Mux     sync.RWMutex
+}
+
+var engines = &engineHolder{
+	Engines: make(map[string]*Engine),
+	Mux:     sync.RWMutex{},
+}

model_migration.go

@@ -0,0 +1,187 @@
+package orm
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+)
+
+type schemaSnapshotColumn struct {
+	Document       `d:"table:__schemas"`
+	ID             int64 `d:"pk"`
+	ModelName      string
+	FieldName      string
+	FieldType      string
+	FieldIndex     int
+	IsRelationship bool
+	IsSynthetic    bool
+}
+
+func (m *Model) toSnapshotColumns() (ssc []*schemaSnapshotColumn) {
+	for _, field := range m.Fields {
+		ssc = append(ssc, &schemaSnapshotColumn{
+			ModelName:  m.Name,
+			FieldName:  field.Name,
+			FieldType:  field.Type.String(),
+			FieldIndex: field.Index,
+		})
+	}
+	for _, rel := range m.Relationships {
+		rt := rel.RelatedType
+		if rel.Kind == reflect.Slice {
+			rt = reflect.SliceOf(rel.RelatedType)
+		}
+		ssc = append(ssc, &schemaSnapshotColumn{
+			ModelName:      m.Name,
+			FieldName:      rel.FieldName,
+			FieldType:      rt.String(),
+			FieldIndex:     rel.Idx,
+			IsRelationship: true,
+			IsSynthetic:    rel.Idx < 0,
+		})
+	}
+	return
+}
+
+func (m *Model) createTableSql() string {
+	var fields []string
+	for _, field := range m.Fields {
+		if !field.isAnonymous() {
+			isStructOrSliceOfStructs := field.Type.Kind() == reflect.Struct ||
+				((field.Type.Kind() == reflect.Slice || field.Type.Kind() == reflect.Array) &&
+					field.Type.Elem().Kind() == reflect.Struct)
+			if field.PrimaryKey {
+				fields = append(fields, fmt.Sprintf("%s %s PRIMARY KEY", field.ColumnName, field.ColumnType))
+			} else if !isStructOrSliceOfStructs || field.ColumnType != "" {
+				lalala := fmt.Sprintf("%s %s", field.ColumnName, field.ColumnType)
+				if !field.Nullable {
+					lalala += " NOT NULL"
+				}
+				lalala += fmt.Sprintf(" DEFAULT %v", defaultColumnValue(field.Type))
+
+				fields = append(fields, lalala)
+			}
+		} else {
+			ft := field.Type
+			for ft.Kind() == reflect.Pointer {
+				ft = ft.Elem()
+			}
+			for i := range ft.NumField() {
+				efield := field.Type.Field(i)
+				ctype := columnType(efield.Type, false, false)
+				if ctype != "" {
+					def := fmt.Sprintf("%s %s NOT NULL DEFAULT %v", pascalToSnakeCase(efield.Name), ctype, defaultColumnValue(efield.Type))
+					fields = append(fields, def)
+				}
+			}
+		}
+	}
+
+	inter := strings.Join(fields, ", ")
+	return fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s);",
+		m.TableName, inter)
+}
+
+func (m *Model) createJoinTableSql(relName string) string {
+	ref, ok := m.Relationships[relName]
+	if !ok {
+		return ""
+	}
+	aTable := m.TableName
+	joinTableName := ref.ComputeJoinTable()
+	fct := serialToRegular(ref.primaryID().ColumnType)
+	rct := serialToRegular(ref.relatedID().ColumnType)
+	pkSection := fmt.Sprintf(",\nPRIMARY KEY (%s_id, %s_id)",
+		aTable,
+		ref.RelatedModel.TableName,
+	)
+	if ref.m2mIsh() {
+		pkSection = ""
+	}
+	return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
+%s_id %s REFERENCES %s(%s) ON DELETE CASCADE,
+%s_id %s REFERENCES %s(%s) ON DELETE CASCADE %s
+);`,
+		joinTableName,
+		ref.Model.TableName,
+		fct,
+		ref.Model.TableName, ref.Model.Fields[ref.Model.IDField].ColumnName,
+		ref.RelatedModel.TableName,
+		rct,
+		ref.RelatedModel.TableName, ref.RelatedModel.Fields[ref.RelatedModel.IDField].ColumnName,
+		pkSection,
+	)
+}
+
+func (m *Model) generateConstraints(engine *Engine) error {
+	for _, rel := range m.Relationships {
+		field := rel.relatedID()
+		if rel.Type != ManyToMany && rel.Type != HasMany && !rel.m2mIsh() {
+			colType := serialToRegular(field.ColumnType)
+			if !field.Nullable && !rel.Nullable {
+				colType += " NOT NULL"
+			}
+			/*constraint := fmt.Sprintf("%s %s REFERENCES %s(%s)",
+				pascalToSnakeCase(rel.joinField()), colType,
+				rel.RelatedModel.TableName,
+				field.ColumnName)
+			if rel.Type != ManyToOne && rel.Type != BelongsTo {
+				constraint += " ON DELETE CASCADE ON UPDATE CASCADE"
+			}*/
+			fk := fmt.Sprintf("fk_%s", pascalToSnakeCase(capitalizeFirst(rel.Model.Name)+rel.FieldName+rel.relatedID().Name))
+			q := fmt.Sprintf(`ALTER TABLE %s
+ADD COLUMN IF NOT EXISTS %s %s,
+ADD CONSTRAINT %s
+FOREIGN KEY (%s) REFERENCES %s(%s)
+ON DELETE CASCADE
+ON UPDATE CASCADE;`,
+				rel.Model.TableName,
+				pascalToSnakeCase(rel.joinField()), colType,
+				fk,
+				pascalToSnakeCase(rel.joinField()),
+				rel.RelatedModel.TableName, field.ColumnName,
+			)
+			dq := fmt.Sprintf(`ALTER TABLE %s DROP CONSTRAINT IF EXISTS %s;`, m.TableName, fk)
+			engine.logSql("drop constraint", dq)
+			engine.logSql("alter table", q)
+			if _, err := engine.conn.Exec(engine.ctx, dq); err != nil {
+				return err
+			}
+			if _, err := engine.conn.Exec(engine.ctx, q); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (m *Model) migrate(engine *Engine) error {
+	sql := m.createTableSql()
+	engine.logSql("create table", sql)
+	if !engine.dryRun {
+		_, err := engine.conn.Exec(engine.ctx, sql)
+		if err != nil {
+			return err
+		}
+	}
+	for relName, rel := range m.Relationships {
+		relkey := rel.ComputeJoinTable()
+		if (rel.Type == ManyToMany && !engine.m2mSeen[relkey]) ||
+			(rel.Model.embeddedIsh && !rel.RelatedModel.embeddedIsh && rel.Type == HasMany) {
+			if rel.Type == ManyToMany {
+				engine.m2mSeen[relkey] = true
+				engine.m2mSeen[rel.Model.Name] = true
+				engine.m2mSeen[rel.RelatedModel.Name] = true
+			}
+			jtsql := m.createJoinTableSql(relName)
+			engine.logSql("crate join table", jtsql)
+			if !engine.dryRun {
+				_, err := engine.conn.Exec(engine.ctx, jtsql)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return m.generateConstraints(engine)
+}

query.go

@@ -0,0 +1,322 @@
+package orm
+
+import (
+	"context"
+	"fmt"
+	sb "github.com/henvic/pgq"
+	"github.com/jackc/pgx/v5"
+	"reflect"
+	"strings"
+)
+
+// Query - contains the state and other details
+// pertaining to the current FUCK operation (Find, Update/Create, Kill [Delete])
+type Query struct {
+	engine         *Engine          // the Engine instance that created this Query
+	model          *Model           // the primary Model this Query pertains to
+	tx             pgx.Tx           // the transaction for insert, update and delete operations
+	ctx            context.Context  // does nothing, but is needed by some pgx functions
+	populationTree map[string]any   // a tree-like map representing the dot-separated paths of fields to populate
+	wheres         map[string][]any // a mapping of where clauses to a list of their arguments
+	joins          []string         // slice of tables to join on before executing Find. useful for hwne you have Where clauses referencing fields/columns in other structs/tables
+	orders         []string         // slice of `ORDER BY` clauses
+	limit          int              // argument to a LIMIT clause, if non-zero
+	offset         int              // unused (for now)
+}
+
+func (q *Query) setModel(val any) *Query {
+	tt := reflect.TypeOf(val)
+	for tt.Kind() == reflect.Ptr {
+		tt = tt.Elem()
+	}
+	q.model = q.engine.modelMap.Map[tt.Name()]
+	return q
+}
+
+func (q *Query) cleanupTx() {
+	q.tx.Rollback(q.ctx)
+	q.tx = nil
+}
+
+// Order - add an `ORDER BY` clause to the current Query.
+// Only applicable for Find queries
+func (q *Query) Order(order string) *Query {
+	q.orders = append(q.orders, order)
+	return q
+}
+
+// Limit - limit resultset to at most `limit` results.
+// does nothing if `limit` <= 0 or the final operation isn't Find
+func (q *Query) Limit(limit int) *Query {
+	if limit > -1 {
+		q.limit = limit
+	}
+	return q
+}
+
+// Offset - skip to the nth result, where n = `offset`.
+// does nothing if `offset` <= 0 or the final operation isn't Find
+func (q *Query) Offset(offset int) *Query {
+	if offset > -1 {
+		q.offset = offset
+	}
+	return q
+}
+
+// Where - add a `WHERE` clause to this query.
+// struct field names can be passed to this method,
+// and they will be automatically converted
+func (q *Query) Where(cond string, args ...any) *Query {
+	q.processWheres(cond, "eq", args...)
+	return q
+}
+
+// WhereRaw - add a `WHERE` clause to this query, except `cond` is passed as-is.
+func (q *Query) WhereRaw(cond string, args ...any) *Query {
+	q.wheres[cond] = args
+	return q
+}
+
+// In - add a `WHERE ... IN(...)` clause to this query
+func (q *Query) In(cond string, args ...any) *Query {
+	q.processWheres(cond, "in", args...)
+	return q
+}
+
+// Join - join the current model's table with the table
+// representing the type of struct field named `field`.
+// Must be called before Where if referencing other
+// structs/types to avoid errors
+func (q *Query) Join(field string) *Query {
+	var clauses []string
+	parts := strings.Split(field, ".")
+	cur := q.model
+	found := false
+	aliasMap := q.getNestedAliases(field)
+
+	for _, part := range parts {
+		rel, ok := cur.Relationships[part]
+		if !ok {
+			found = false
+			break
+		}
+		if rel.FieldName != part {
+			found = false
+			break
+		}
+		found = true
+		aliases := aliasMap[rel]
+		curAlias := aliases[0]
+		nalias := aliases[1]
+		if rel.m2mIsh() || rel.Type == ManyToMany {
+			joinAlias := aliases[2]
+			jc1 := fmt.Sprintf("%s AS %s ON %s.%s = %s.%s_id",
+				rel.ComputeJoinTable(), joinAlias,
+				curAlias, cur.idField().ColumnName,
+				joinAlias, rel.Model.TableName,
+			)
+			jc2 := fmt.Sprintf("%s AS %s ON %s.%s_id = %s.%s",
+				rel.RelatedModel.TableName, nalias,
+				joinAlias, rel.RelatedModel.TableName,
+				nalias, rel.relatedID().ColumnName,
+			)
+			clauses = append(clauses, jc1, jc2)
+		}
+		if rel.Type == HasMany || rel.Type == HasOne {
+			fkr := rel.RelatedModel.Relationships[cur.Name]
+			if fkr != nil {
+				jc := fmt.Sprintf("%s AS %s ON %s.%s = %s.%s",
+					rel.RelatedModel.TableName, nalias,
+					curAlias, cur.idField().ColumnName,
+					nalias, pascalToSnakeCase(fkr.joinField()),
+				)
+				clauses = append(clauses, jc)
+			}
+		}
+		if rel.Type == BelongsTo {
+			jc := fmt.Sprintf("%s AS %s ON %s.%s = %s.%s",
+				rel.RelatedModel.TableName, nalias,
+				curAlias, pascalToSnakeCase(rel.joinField()),
+				nalias, rel.RelatedModel.idField().ColumnName,
+			)
+			clauses = append(clauses, jc)
+		}
+		curAlias = nalias
+		cur = rel.RelatedModel
+	}
+	if found {
+		q.joins = append(q.joins, clauses...)
+	}
+	return q
+}
+
+func (q *Query) getNestedAliases(field string) (amap map[*Relationship][]string) {
+	amap = make(map[*Relationship][]string)
+	parts := strings.Split(field, ".")
+	cur := q.model
+	curAlias := q.model.TableName
+	first := curAlias
+	found := false
+
+	for _, part := range parts {
+		rel, ok := cur.Relationships[part]
+		if !ok {
+			found = false
+			break
+		}
+		if rel.FieldName != part {
+			found = false
+			break
+		}
+		found = true
+		amap[rel] = make([]string, 0)
+
+		nalias := pascalToSnakeCase(part)
+		if rel.m2mIsh() || rel.Type == ManyToMany {
+			joinAlias := rel.ComputeJoinTable() + "_joined"
+			amap[rel] = append(amap[rel], curAlias, nalias, joinAlias)
+		} else if rel.Type == HasMany || rel.Type == HasOne || rel.Type == BelongsTo {
+			amap[rel] = append(amap[rel], curAlias, nalias)
+		}
+
+		curAlias = nalias
+		cur = rel.RelatedModel
+	}
+	if !found {
+		return
+	}
+	amap[nil] = []string{first}
+	return
+}
+
+func (q *Query) processWheres(cond string, exprKind string, args ...any) {
+	parts := strings.SplitN(cond, " ", 2)
+	var translatedColumn string
+	fieldPath := parts[0]
+	ncond := ""
+	if len(parts) > 1 {
+		ncond = " " + parts[1]
+	}
+	pathParts := strings.Split(fieldPath, ".")
+	if len(pathParts) > 1 {
+		relPath := pathParts[:len(pathParts)-1]
+		fieldName := pathParts[len(pathParts)-1]
+		relPathStr := strings.Join(relPath, ".")
+		aliasMap := q.getNestedAliases(relPathStr)
+		for r, a := range aliasMap {
+			if r == nil {
+				continue
+			}
+			f, ok := r.RelatedModel.Fields[fieldName]
+			if ok {
+				translatedColumn = fmt.Sprintf("%s.%s", a[1], f.ColumnName)
+			}
+		}
+	} else if pf := q.model.Fields[pathParts[0]]; pf != nil {
+		translatedColumn = fmt.Sprintf("%s.%s", q.model.TableName, pf.ColumnName)
+	}
+	var tq string
+	switch strings.ToLower(exprKind) {
+	case "in":
+		tq = fmt.Sprintf("%s IN (%s)", translatedColumn, MakePlaceholders(len(args)))
+	default:
+		tq = fmt.Sprintf("%s%s", translatedColumn, ncond)
+	}
+	q.wheres[tq] = args
+}
+
+// buildSQL - aggregates the information in this Query into a pgq.SelectBuilder.
+// it returns a slice of column names as well to avoid issues with scanning
+func (q *Query) buildSQL() (cols []string, anonymousCols map[string][]string, finalSb sb.SelectBuilder, err error) {
+	var inParents []any
+	anonymousCols = make(map[string][]string)
+	for _, field := range q.model.Fields {
+		if field.isAnonymous() {
+			for _, ef := range field.embeddedFields {
+				anonymousCols[field.ColumnName] = append(anonymousCols[field.ColumnName], ef.ColumnName)
+			}
+			continue
+		}
+		cols = append(cols, field.ColumnName)
+	}
+	finalSb = sb.Select(cols...)
+	for _, cc := range anonymousCols {
+		finalSb = finalSb.Columns(cc...)
+	}
+	finalSb = finalSb.From(q.model.TableName)
+	if len(q.joins) > 0 {
+		idq := sb.Select(fmt.Sprintf("%s.%s", q.model.TableName, q.model.idField().ColumnName)).
+			Distinct().
+			From(q.model.TableName)
+		for w, arg := range q.wheres {
+			idq = idq.Where(w, arg...)
+		}
+		for _, j := range q.joins {
+			idq = idq.Join(j)
+		}
+		qq, qa := idq.MustSQL()
+		var rows pgx.Rows
+		rows, err = q.engine.conn.Query(q.ctx, qq, qa...)
+		if err != nil {
+			return
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var id any
+			if err = rows.Scan(&id); err != nil {
+				return
+			}
+			inParents = append(inParents, id)
+		}
+		if len(inParents) == 0 {
+			return
+		}
+	}
+	if len(inParents) > 0 {
+		finalSb = finalSb.Where(
+			fmt.Sprintf("%s IN (%s)",
+				q.model.idField().ColumnName,
+				MakePlaceholders(len(inParents))), inParents...)
+	} else if len(q.wheres) > 0 {
+		for k, vv := range q.wheres {
+			finalSb = finalSb.Where(k, vv...)
+		}
+	}
+ool:
+	for _, o := range q.orders {
+		ac, ok := q.model.Fields[o]
+		if !ok {
+			var rel = q.model.Relationships[o]
+
+			if rel != nil {
+				if strings.Contains(o, ".") {
+					split := strings.Split(strings.TrimSuffix(strings.TrimPrefix(o, "."), "."), ".")
+					cm := rel.Model
+					for i, s := range split {
+						if rel != nil {
+							cm = rel.RelatedModel
+						} else if i == len(split)-1 {
+							break
+						} else {
+							continue ool
+						}
+						rel = cm.Relationships[s]
+					}
+					lf := split[len(split)-1]
+					ac, ok = cm.Fields[lf]
+					if !ok {
+						continue
+					}
+				}
+			}
+		}
+		finalSb = finalSb.OrderBy(ac.ColumnName)
+	}
+	if q.limit > 0 {
+		finalSb = finalSb.Limit(uint64(q.limit))
+	}
+	if q.offset > 0 {
+		finalSb = finalSb.Offset(uint64(q.offset))
+	}
+	return
+}

query_populate.go

@@ -0,0 +1,400 @@
+package orm
+
+import (
+	"fmt"
+	sb "github.com/henvic/pgq"
+	"github.com/jackc/pgx/v5"
+	"reflect"
+	"strings"
+)
+
+const PopulateAll = "~~~ALL~~~"
+
+// Populate - allows you to pre-load embedded structs/slices within the current model.
+// use dots between field names to specify nested paths. use the PopulateAll constant to populate all
+// relationships non-recursively
+func (q *Query) Populate(fields ...string) *Query {
+	if q.populationTree == nil {
+		q.populationTree = make(map[string]any)
+	}
+	for _, field := range fields {
+		if field == PopulateAll {
+			for k := range q.model.Relationships {
+				if _, ok := q.populationTree[k]; !ok {
+					q.populationTree[k] = make(map[string]any)
+				}
+			}
+			continue
+		}
+		cur := q.populationTree
+		parts := strings.Split(field, ".")
+		for _, part := range parts {
+			if _, ok := cur[part]; !ok {
+				cur[part] = make(map[string]any)
+			}
+			cur = cur[part].(map[string]any)
+		}
+	}
+	return q
+}
+
+func (q *Query) processPopulate(parent reflect.Value, model *Model, populationTree map[string]any) error {
+	if parent.Len() == 0 {
+		return nil
+	}
+	pids := make([]any, 0)
+	var err error
+	idField := model.IDField
+	for i := range parent.Len() {
+		pval := parent.Index(i)
+		if pval.Kind() == reflect.Pointer {
+			pval = pval.Elem()
+		}
+		pids = append(pids, pval.FieldByName(idField).Interface())
+	}
+	toClose := make([]pgx.Rows, 0)
+	defer func() {
+		for _, c := range toClose {
+			c.Close()
+		}
+	}()
+	for p, nested := range populationTree {
+		var rel *Relationship
+		for _, r := range model.Relationships {
+			if r.FieldName == p {
+				rel = r
+				break
+			}
+		}
+		if rel == nil {
+			return fmt.Errorf("field '%s' not found in model '%s'", p, model.Name)
+		}
+
+		childSlice := reflect.Value{}
+
+		if (rel.Type == HasMany || rel.Type == HasOne) && !rel.m2mIsh() {
+			childSlice, err = q.populateHas(rel, parent, pids)
+		} else if rel.Type == BelongsTo {
+			childSlice, err = q.populateBelongsTo(rel, parent, pids)
+		} else if rel.Type == ManyToMany || rel.m2mIsh() {
+			childSlice, err = q.populateManyToMany(rel, parent, pids)
+		}
+		if err != nil {
+			return fmt.Errorf("failed to populate field at '%s': %w", p, err)
+		}
+		ntree, ok := nested.(map[string]any)
+		if ok && len(ntree) > 0 && childSlice.IsValid() && childSlice.Len() > 0 {
+			if err = q.processPopulate(childSlice, rel.RelatedModel, ntree); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (q *Query) populateHas(rel *Relationship, parent reflect.Value, parentIds []any) (reflect.Value, error) {
+	fkf := rel.primaryID()
+	var fk string
+	if fkf != nil && fkf.ColumnType != "" {
+		fk = fkf.ColumnName
+	} else if rel.relatedID() != nil {
+		fk = pascalToSnakeCase(rel.RelatedModel.Name + rel.relatedID().Name)
+	}
+
+	if rel.RelatedModel.embeddedIsh && !rel.Model.embeddedIsh && rel.Type == HasMany {
+		arel := rel.RelatedModel.Relationships[rel.Model.Name]
+		fk = pascalToSnakeCase(arel.joinField())
+	}
+	ccols := make([]string, 0)
+	anonymousCols := make(map[string]map[string]*Field)
+	for _, f := range rel.RelatedModel.Fields {
+		if !f.isAnonymous() {
+			ccols = append(ccols, f.ColumnName)
+		}
+	}
+	for _, f := range rel.RelatedModel.Fields {
+		if f.isAnonymous() {
+			ccols = append(ccols, f.anonymousColumnNames()...)
+			anonymousCols[f.Name] = f.embeddedFields
+		}
+	}
+	for _, r := range rel.RelatedModel.Relationships {
+		if r.Type != ManyToOne {
+			continue
+		}
+		ccols = append(ccols, pascalToSnakeCase(r.joinField()))
+	}
+	/*var tableName string
+	if rel.Type == HasOne {
+		tableName = rel.Model.TableName
+	}
+	if rel.Type == HasMany {
+		tableName = rel.RelatedModel.TableName
+	}*/
+	aq, aa := sb.Select(ccols...).
+		From(rel.RelatedModel.TableName).
+		Where(fmt.Sprintf("%s IN (%s)", fk, MakePlaceholders(len(parentIds))), parentIds...).MustSQL()
+	q.engine.logQuery("populate", aq, aa)
+	rows, err := q.engine.conn.Query(q.ctx, aq, aa...)
+	if err != nil {
+		return reflect.Value{}, err
+	}
+	defer rows.Close()
+	idFieldName := rel.Model.IDField
+	idField := rel.Model.Fields[idFieldName]
+	if rel.Type == HasMany {
+		childMap := reflect.MakeMap(reflect.MapOf(
+			idField.Type,
+			reflect.SliceOf(rel.RelatedModel.Type),
+		))
+		for rows.Next() {
+			child := reflect.New(rel.RelatedModel.Type).Elem()
+			var fkValue any
+			scanDest, _ := buildScanDest(child, rel.RelatedModel, rel, ccols, anonymousCols, &fkValue)
+			if err = rows.Scan(scanDest...); err != nil {
+				return reflect.Value{}, err
+			}
+			fkVal := reflect.ValueOf(fkValue)
+			childrenOfParent := childMap.MapIndex(fkVal)
+			if !childrenOfParent.IsValid() {
+				childrenOfParent = reflect.MakeSlice(reflect.SliceOf(rel.RelatedModel.Type), 0, 0)
+			}
+			childrenOfParent = reflect.Append(childrenOfParent, child)
+			childMap.SetMapIndex(fkVal, childrenOfParent)
+		}
+		for i := range parent.Len() {
+			ps := parent.Index(i)
+			if ps.Kind() == reflect.Pointer {
+				ps = ps.Elem()
+			}
+			pid := ps.FieldByName(idFieldName)
+			c := childMap.MapIndex(pid)
+			if c.IsValid() {
+				ps.FieldByName(rel.FieldName).Set(c)
+			}
+		}
+	} else {
+		childMap := reflect.MakeMap(reflect.MapOf(idField.Type, rel.RelatedModel.Type))
+		for rows.Next() {
+			child := reflect.New(rel.RelatedModel.Type).Elem()
+			var fkValue any
+			scanDest, _ := buildScanDest(child, rel.Model, rel, ccols, anonymousCols, &fkValue)
+			if err = rows.Scan(scanDest...); err != nil {
+				return reflect.Value{}, err
+			}
+			fkVal := reflect.ValueOf(fkValue)
+			childMap.SetMapIndex(fkVal, child)
+		}
+		for i := range parent.Len() {
+			ps := parent.Index(i)
+			if ps.Kind() == reflect.Pointer {
+				ps = ps.Elem()
+			}
+			parentID := ps.FieldByName(idFieldName)
+			if child := childMap.MapIndex(parentID); child.IsValid() {
+				ps.FieldByName(rel.FieldName).Set(child)
+			}
+		}
+	}
+
+	childSlice := reflect.MakeSlice(reflect.SliceOf(reflect.PointerTo(rel.RelatedModel.Type)), 0, 0)
+	for i := range parent.Len() {
+		ps := parent.Index(i)
+		if ps.Kind() == reflect.Ptr {
+			ps = ps.Elem()
+		}
+		childField := ps.FieldByName(rel.FieldName)
+		if !childField.IsValid() {
+			continue
+		}
+		if rel.Type == HasMany {
+			for j := range childField.Len() {
+				childSlice = reflect.Append(childSlice, childField.Index(j).Addr())
+			}
+		} else {
+			if !childField.IsZero() {
+				childSlice = reflect.Append(childSlice, childField.Addr())
+			}
+		}
+	}
+	return childSlice, nil
+}
+
+func (q *Query) populateManyToMany(rel *Relationship, parent reflect.Value, parentIds []any) (reflect.Value, error) {
+	inPlaceholders := MakePlaceholders(len(parentIds))
+	ccols := make([]string, 0)
+	anonymousCols := make(map[string]map[string]*Field)
+	for _, f := range rel.RelatedModel.Fields {
+		if !f.isAnonymous() {
+			ccols = append(ccols, "m."+f.ColumnName)
+		}
+	}
+	for _, f := range rel.RelatedModel.Fields {
+		if f.isAnonymous() {
+			for ecol := range f.embeddedFields {
+				ccols = append(ccols, "m."+ecol)
+			}
+			anonymousCols[f.Name] = f.embeddedFields
+		}
+	}
+	ccols = append(ccols, fmt.Sprintf("jt.%s_id", rel.Model.TableName))
+
+	mq, ma := sb.Select(ccols...).
+		From(fmt.Sprintf("%s AS m", rel.RelatedModel.TableName)).
+		Join(
+			fmt.Sprintf("%s AS jt ON m.%s = jt.%s_id",
+				rel.ComputeJoinTable(),
+				rel.relatedID().ColumnName, rel.RelatedModel.TableName)).
+		Where(fmt.Sprintf("jt.%s_id IN (%s)",
+			rel.Model.TableName, inPlaceholders), parentIds...).MustSQL()
+	q.engine.logQuery("populate/join", mq, ma)
+	rows, err := q.engine.conn.Query(q.ctx, mq, ma...)
+	if err != nil {
+		return reflect.Value{}, err
+	}
+	defer rows.Close()
+	idFieldName := rel.Model.IDField
+	idField := rel.Model.Fields[idFieldName]
+	childMap := reflect.MakeMap(reflect.MapOf(
+		idField.Type,
+		reflect.SliceOf(rel.RelatedModel.Type)))
+	for rows.Next() {
+		child := reflect.New(rel.RelatedModel.Type).Elem()
+		var foreignKeyValue any
+		scanDest, _ := buildScanDest(child, rel.RelatedModel, rel, ccols, anonymousCols, &foreignKeyValue)
+		if err = rows.Scan(scanDest...); err != nil {
+			return reflect.Value{}, err
+		}
+		fkVal := reflect.ValueOf(foreignKeyValue)
+		childrenOfParent := childMap.MapIndex(fkVal)
+		if !childrenOfParent.IsValid() {
+			childrenOfParent = reflect.MakeSlice(reflect.SliceOf(rel.RelatedModel.Type), 0, 0)
+		}
+		childrenOfParent = reflect.Append(childrenOfParent, child)
+		childMap.SetMapIndex(fkVal, childrenOfParent)
+	}
+	for i := range parent.Len() {
+		p := parent.Index(i)
+		if p.Kind() == reflect.Ptr {
+			p = p.Elem()
+		}
+		parentID := p.FieldByName(rel.primaryID().Name)
+		if children := childMap.MapIndex(parentID); children.IsValid() {
+			p.FieldByName(rel.FieldName).Set(children)
+		}
+	}
+	childSlice := reflect.MakeSlice(reflect.SliceOf(reflect.PointerTo(rel.RelatedModel.Type)), 0, 0)
+	for i := range parent.Len() {
+		ps := parent.Index(i)
+		if ps.Kind() == reflect.Ptr {
+			ps = ps.Elem()
+		}
+		childField := ps.FieldByName(rel.FieldName)
+		if childField.IsValid() {
+			for j := range childField.Len() {
+				childSlice = reflect.Append(childSlice, childField.Index(j).Addr())
+			}
+		}
+	}
+	return childSlice, nil
+}
+
+func (q *Query) populateBelongsTo(rel *Relationship, childrenSlice reflect.Value, childIDs []any) (reflect.Value, error) {
+	childIdField := rel.Model.Fields[rel.Model.IDField]
+	parentIdField := rel.RelatedModel.Fields[rel.RelatedModel.IDField]
+	fk := pascalToSnakeCase(rel.joinField())
+	qs, qa := sb.Select(childIdField.ColumnName, fk).
+		From(rel.Model.TableName).
+		Where(fmt.Sprintf("%s IN (%s)",
+			childIdField.ColumnName, MakePlaceholders(len(childIDs)),
+		), childIDs...).MustSQL()
+	q.engine.logQuery("populate/belongs-to", qs, qa)
+	rows, err := q.engine.conn.Query(q.ctx, qs, qa...)
+	if err != nil {
+		return reflect.Value{}, err
+	}
+	childParentKeyMap := make(map[any]any)
+	parentKeyValues := make([]any, 0)
+	parentKeySet := make(map[any]bool)
+	for rows.Next() {
+		var cid, pfk any
+		err = rows.Scan(&cid, &pfk)
+		if err != nil {
+			rows.Close()
+			return reflect.Value{}, err
+		}
+		childParentKeyMap[cid] = pfk
+		if !parentKeySet[pfk] {
+			parentKeySet[pfk] = true
+			parentKeyValues = append(parentKeyValues, pfk)
+		}
+	}
+	rows.Close()
+	if len(parentKeyValues) == 0 {
+		return reflect.Value{}, nil
+	}
+	pcols := make([]string, 0)
+	anonymousCols := make(map[string]map[string]*Field)
+	for _, f := range rel.RelatedModel.Fields {
+		if !f.isAnonymous() {
+			pcols = append(pcols, f.ColumnName)
+		}
+	}
+	for _, f := range rel.RelatedModel.Fields {
+		if f.isAnonymous() {
+			pcols = append(pcols, f.anonymousColumnNames()...)
+			anonymousCols[f.Name] = f.embeddedFields
+		}
+	}
+	pquery, pqargs := sb.Select(pcols...).
+		From(rel.RelatedModel.TableName).
+		Where(fmt.Sprintf("%s IN (%s)",
+			parentIdField.ColumnName,
+			MakePlaceholders(len(parentKeyValues))), parentKeyValues...).
+		MustSQL()
+	q.engine.logQuery("populate/belongs-to->parent", pquery, pqargs)
+	parentRows, err := q.engine.conn.Query(q.ctx, pquery, pqargs...)
+	if err != nil {
+		return reflect.Value{}, err
+	}
+	defer parentRows.Close()
+	parentMap := reflect.MakeMap(reflect.MapOf(
+		parentIdField.Type,
+		rel.RelatedModel.Type,
+	))
+	for parentRows.Next() {
+		parent := reflect.New(rel.RelatedModel.Type).Elem()
+		scanDst, _ := buildScanDest(parent, rel.RelatedModel, rel, pcols, anonymousCols, nil)
+		if err = parentRows.Scan(scanDst...); err != nil {
+			return reflect.Value{}, err
+		}
+		parentId := parent.FieldByName(rel.RelatedModel.IDField)
+		parentMap.SetMapIndex(parentId, parent)
+	}
+	for i := range childrenSlice.Len() {
+		child := childrenSlice.Index(i)
+		childID := child.FieldByName(rel.Model.IDField)
+		if parentKey, ok := childParentKeyMap[childID.Interface()]; ok && parentKey != nil {
+			if parent := parentMap.MapIndex(reflect.ValueOf(parentKey)); parent.IsValid() {
+				child.FieldByName(rel.FieldName).Set(parent)
+			}
+		}
+	}
+	ntype := rel.RelatedModel.Type
+	if rel.Kind == reflect.Pointer {
+		ntype = reflect.PointerTo(rel.RelatedModel.Type)
+	}
+	parentSlice := reflect.MakeSlice(reflect.SliceOf(reflect.PointerTo(ntype)), 0, 0)
+	for i := range childrenSlice.Len() {
+		ps := childrenSlice.Index(i)
+		if ps.Kind() == reflect.Ptr {
+			ps = ps.Elem()
+		}
+		childField := ps.FieldByName(rel.FieldName)
+		if childField.IsValid() {
+			parentSlice = reflect.Append(parentSlice, childField.Addr())
+		}
+	}
+	return parentSlice, nil
+}

query_tail.go

@@ -0,0 +1,369 @@
+package orm
+
+import (
+	"fmt"
+	sb "github.com/henvic/pgq"
+	"github.com/jackc/pgx/v5"
+	"reflect"
+	"time"
+)
+
+// Find - transpiles this query into SQL and places the result in `dest`
+func (q *Query) Find(dest any) error {
+	dstVal := reflect.ValueOf(dest)
+	if dstVal.Kind() != reflect.Ptr {
+		return fmt.Errorf("destination must be a pointer, got: %v", dstVal.Kind())
+	}
+	maybeSlice := dstVal.Elem()
+	cols, acols, sqlb, err := q.buildSQL()
+	if err != nil {
+		return err
+	}
+	qq, qa := sqlb.MustSQL()
+	q.engine.logQuery("find", qq, qa)
+
+	if maybeSlice.Kind() == reflect.Struct {
+		row := q.engine.conn.QueryRow(q.ctx, qq, qa...)
+		if err = scanRow(row, cols, acols, maybeSlice, q.model); err != nil {
+			return err
+		}
+	} else if maybeSlice.Kind() == reflect.Slice ||
+		maybeSlice.Kind() == reflect.Array {
+		var rows pgx.Rows
+		rows, err = q.engine.conn.Query(q.ctx, qq, qa...)
+		if err != nil {
+			return err
+		}
+		defer rows.Close()
+
+		etype := maybeSlice.Type().Elem()
+		for rows.Next() {
+			nelem := reflect.New(etype).Elem()
+			if err = scanRow(rows, cols, acols, nelem, q.model); err != nil {
+				return err
+			}
+			maybeSlice.Set(reflect.Append(maybeSlice, nelem))
+		}
+	} else {
+		return fmt.Errorf("unsupported destination type: %s", maybeSlice.Kind())
+	}
+	if len(q.populationTree) > 0 {
+		nslice := maybeSlice
+		var wasPassedStruct bool
+		if nslice.Kind() == reflect.Struct {
+			nslice = reflect.MakeSlice(reflect.SliceOf(maybeSlice.Type()), 0, 0)
+			wasPassedStruct = true
+			nslice = reflect.Append(nslice, maybeSlice)
+		}
+		err = q.processPopulate(nslice, q.model, q.populationTree)
+		if err == nil && wasPassedStruct {
+			maybeSlice.Set(nslice.Index(0))
+		}
+		return err
+	}
+	return nil
+}
+
+// Save - create or update `val` in the database
+func (q *Query) Save(val any) error {
+	return q.saveOrCreate(val, false)
+}
+
+// Create - like Save, but hints to the query processor that you want to insert, not update.
+// useful if you're importing data and want to keep the IDs intact.
+func (q *Query) Create(val any) error {
+	return q.saveOrCreate(val, true)
+}
+
+// UpdateRaw - takes a mapping of struct field names to
+// SQL expressions, updating each field's associated column accordingly
+func (q *Query) UpdateRaw(values map[string]any) (int64, error) {
+	var err error
+	var subQuery sb.SelectBuilder
+	stmt := sb.Update(q.model.TableName)
+	_, _, subQuery, err = q.buildSQL()
+	if err != nil {
+		return 0, err
+	}
+	subQuery = sb.Select(q.model.idField().ColumnName).FromSelect(subQuery, "subQuery")
+	stmt = stmt.Where(wrapQueryIn(subQuery,
+		q.model.idField().ColumnName))
+	for k, v := range values {
+		asString, isString := v.(string)
+		if f, ok := q.model.Fields[k]; ok {
+			if isString {
+				stmt = stmt.Set(f.ColumnName, sb.Expr(asString))
+			} else {
+				stmt = stmt.Set(f.ColumnName, v)
+			}
+		}
+		if _, ok := q.model.FieldsByColumnName[k]; ok {
+			if isString {
+				stmt = stmt.Set(k, sb.Expr(asString))
+			} else {
+				stmt = stmt.Set(k, v)
+			}
+		}
+	}
+	sql, args := stmt.MustSQL()
+	q.engine.logQuery("update/raw", sql, args)
+	q.tx, err = q.engine.conn.Begin(q.ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer q.cleanupTx()
+
+	ctag, err := q.tx.Exec(q.ctx, sql, args...)
+	if err != nil {
+		return 0, err
+	}
+	return ctag.RowsAffected(), q.tx.Commit(q.ctx)
+}
+
+// Delete - delete one or more entities matching previous conditions specified
+// by methods like Where, WhereRaw, or In. will refuse to execute if no
+// conditions were specified for safety reasons. to override this, call
+// WhereRaw("true") or WhereRaw("1 = 1") before this method.
+func (q *Query) Delete() (int64, error) {
+	var err error
+	var subQuery sb.SelectBuilder
+	if len(q.wheres) < 1 {
+		return 0, ErrNoConditionOnDeleteOrUpdate
+	}
+	q.tx, err = q.engine.conn.Begin(q.ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer q.cleanupTx()
+	_, _, subQuery, err = q.buildSQL()
+	if err != nil {
+		return 0, err
+	}
+
+	sqlb := sb.Delete(q.model.TableName).Where(subQuery)
+	sql, sqla := sqlb.MustSQL()
+	q.engine.logQuery("delete", sql, sqla)
+	cmdTag, err := q.tx.Exec(q.ctx, sql, sqla...)
+	if err != nil {
+		return 0, fmt.Errorf("failed to delete: %w", err)
+	}
+	return cmdTag.RowsAffected(), nil
+}
+
+func (q *Query) saveOrCreate(val any, shouldCreate bool) error {
+	v := reflect.ValueOf(val)
+	if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
+		return fmt.Errorf("Save() must be called with a pointer to a struct")
+	}
+	var err error
+	q.tx, err = q.engine.conn.BeginTx(q.ctx, pgx.TxOptions{
+		AccessMode: pgx.ReadWrite,
+		IsoLevel:   pgx.ReadUncommitted,
+	})
+	if err != nil {
+		return err
+	}
+	defer q.cleanupTx()
+	if _, err = q.doSave(v.Elem(), q.engine.modelMap.Map[v.Elem().Type().Name()], nil, shouldCreate); err != nil {
+		return err
+	}
+	return q.tx.Commit(q.ctx)
+}
+
+func (q *Query) doSave(val reflect.Value, model *Model, parentFks map[string]any, shouldInsert bool) (any, error) {
+	idField := model.Fields[model.IDField]
+	var pkField reflect.Value
+	if val.Kind() == reflect.Pointer {
+		if !val.Elem().IsValid() || val.Elem().IsZero() {
+			return nil, nil
+		}
+		pkField = val.Elem().FieldByName(model.IDField)
+	} else {
+		pkField = val.FieldByName(model.IDField)
+	}
+	isNew := pkField.IsZero()
+	var exists bool
+	if !pkField.IsZero() {
+		eb := sb.Select("1").
+			Prefix("SELECT EXISTS (").
+			From(model.TableName).
+			Where(fmt.Sprintf("%s = ?", idField.ColumnName), pkField.Interface()).
+			Suffix(")")
+		ebs, eba := eb.MustSQL()
+		var ex bool
+		err := q.tx.QueryRow(q.ctx, ebs, eba...).Scan(&ex)
+		if err != nil {
+			q.engine.logger.Warn("error while checking existence", "err", err.Error())
+		}
+		exists = ex
+	}
+	/*{
+		el, ok := q.seenIds[model]
+		if !ok {
+			q.seenIds[model] = make(map[any]bool)
+		}
+		if ok && el[pkField.Interface()] {
+			return pkField.Interface(), nil
+		}
+		if !isNew {
+			q.seenIds[model][pkField.Interface()] = true
+		}
+	}*/
+	doInsert := isNew || !exists
+	var cols []string
+	args := make([]any, 0)
+	seenJoinTables := make(map[string]map[any]bool)
+	for _, rel := range model.Relationships {
+		if rel.Type != BelongsTo {
+			continue
+		}
+		parentVal := val.FieldByName(rel.FieldName)
+		if parentVal.IsValid() {
+			nid, err := q.doSave(parentVal, rel.RelatedModel, nil, rel.RelatedModel.needsPrimaryKey(parentVal) && isNew)
+			if err != nil {
+				return nil, err
+			}
+			cols = append(cols, pascalToSnakeCase(rel.joinField()))
+			args = append(args, nid)
+		} else if parentVal.IsValid() {
+			_, nid := rel.RelatedModel.getPrimaryKey(parentVal)
+			cols = append(cols, pascalToSnakeCase(rel.joinField()))
+			args = append(args, nid)
+		}
+	}
+	for _, ff := range model.Fields {
+		var fv reflect.Value
+		if ff.Index > -1 && !ff.isAnonymous() {
+			fv = val.Field(ff.Index)
+		} else if ff.Index > -1 {
+			for col, ef := range ff.embeddedFields {
+				fv = val.Field(ff.Index)
+				cols = append(cols, col)
+				eif := fv.FieldByName(ef.Name)
+				if ff.Name == documentField && canConvertTo[Document](ff.Type) {
+					asTime, ok := eif.Interface().(time.Time)
+					shouldCreate := ok && (asTime.IsZero() || eif.IsZero())
+					if doInsert && ef.Name == createdField && shouldCreate {
+						eif.Set(reflect.ValueOf(time.Now()))
+					} else if ef.Name == modifiedField || shouldCreate {
+						eif.Set(reflect.ValueOf(time.Now()))
+					}
+					args = append(args, eif.Interface())
+					continue
+				}
+				args = append(args, fv.FieldByName(ef.Name).Interface())
+			}
+			continue
+		}
+		if ff.Name == model.IDField {
+			if !isNew && fv.IsValid() {
+				cols = append(cols, ff.ColumnName)
+				args = append(args, fv.Interface())
+			}
+			continue
+		}
+
+		if fv.IsValid() {
+			cols = append(cols, ff.ColumnName)
+			args = append(args, fv.Interface())
+		}
+	}
+	for k, fk := range parentFks {
+		cols = append(cols, k)
+		args = append(args, fk)
+	}
+	var qq string
+	var qa []any
+	if doInsert {
+		osb := sb.Insert(model.TableName)
+		if len(cols) == 0 {
+			qq = fmt.Sprintf("INSERT INTO %s DEFAULT VALUES RETURNING %s", model.TableName, idField.ColumnName)
+		} else {
+			osb = osb.Columns(cols...).Values(args...)
+			qq, qa = osb.Returning(idField.ColumnName).MustSQL()
+		}
+	} else {
+		osb := sb.Update(model.TableName)
+		for i := range cols {
+			osb = osb.Set(cols[i], args[i])
+		}
+		osb = osb.Where(fmt.Sprintf("%s = ?", idField.ColumnName), pkField.Interface())
+		qq, qa = osb.MustSQL()
+	}
+	if doInsert {
+		var nid any
+		q.engine.logQuery("insert", qq, qa)
+		row := q.tx.QueryRow(q.ctx, qq, qa...)
+		err := row.Scan(&nid)
+		if err != nil {
+			return nil, fmt.Errorf("insert failed for model %s: %w", model.Name, err)
+		}
+		pkField.Set(reflect.ValueOf(nid))
+	} else {
+		q.engine.logQuery("update", qq, qa)
+		_, err := q.tx.Exec(q.ctx, qq, qa...)
+		if err != nil {
+			return nil, fmt.Errorf("update failed for model %s: %w", model.Name, err)
+		}
+	}
+	/*if _, ok := q.seenIds[model]; !ok {
+		q.seenIds[model] = make(map[any]bool)
+	}
+	q.seenIds[model][pkField.Interface()] = true*/
+	for _, rel := range model.Relationships {
+
+		if rel.Idx > -1 && rel.Idx < val.NumField() {
+			fv := val.FieldByName(rel.FieldName)
+			cm := rel.RelatedModel
+			pfks := map[string]any{}
+			if !model.embeddedIsh && rel.Type == HasMany {
+				{
+					rm := cm.Relationships[model.Name]
+					if rm != nil && rm.Type == ManyToOne {
+						pfks[pascalToSnakeCase(rm.joinField())] = pkField.Interface()
+					}
+				}
+				for j := range fv.Len() {
+					child := fv.Index(j).Addr().Elem()
+					if _, err := q.doSave(child, cm, pfks, cm.needsPrimaryKey(child)); err != nil {
+						return nil, err
+					}
+				}
+			} else if rel.Type == HasOne && cm.embeddedIsh {
+				if _, err := q.doSave(fv, cm, pfks, cm.needsPrimaryKey(fv)); err != nil {
+					return nil, err
+				}
+			} else if rel.m2mIsh() || rel.Type == ManyToMany || (model.embeddedIsh && cm.embeddedIsh && rel.Type == HasMany) {
+				if seenJoinTables[rel.ComputeJoinTable()] == nil {
+					seenJoinTables[rel.ComputeJoinTable()] = make(map[any]bool)
+				}
+				if !seenJoinTables[rel.ComputeJoinTable()][pkField.Interface()] {
+					seenJoinTables[rel.ComputeJoinTable()][pkField.Interface()] = true
+					if err := rel.joinDelete(pkField.Interface(), nil, q); err != nil {
+						return nil, fmt.Errorf("error deleting existing association: %w", err)
+					}
+				}
+				if fv.Kind() == reflect.Slice || fv.Kind() == reflect.Array {
+					mField := model.Fields[model.IDField]
+					mpks := map[string]any{}
+					if !model.embeddedIsh {
+						mpks[model.TableName+"_"+mField.ColumnName] = pkField.Interface()
+					}
+					for i := range fv.Len() {
+						cur := fv.Index(i)
+						if _, err := q.doSave(cur, cm, mpks, cm.needsPrimaryKey(cur) && pkField.IsZero()); err != nil {
+							return nil, err
+						}
+
+						if rel.m2mIsh() || rel.Type == ManyToMany {
+							if err := rel.joinInsert(cur, q, pkField.Interface()); err != nil {
+								return nil, fmt.Errorf("failed to insert association for model %s: %w", model.Name, err)
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return pkField.Interface(), nil
+}

relationship.go

@@ -0,0 +1,200 @@
+package orm
+
+import (
+	"fmt"
+	sb "github.com/henvic/pgq"
+	"reflect"
+	"strings"
+)
+
+// RelationshipType - used to distinguish a Relationship between various
+// common entity relationship types
+type RelationshipType int
+
+const (
+	HasOne RelationshipType = iota
+	HasMany
+	BelongsTo
+	ManyToOne // the other side of a HasMany relationship
+	ManyToMany
+)
+
+// Relationship - intermediate representation of how two types
+// relate to each other. i.e., if struct A embeds struct B,
+// a Relationship will be created for those two while parsing the Model for A.
+type Relationship struct {
+	Type         RelationshipType // the type of this relationship (see RelationshipType)
+	JoinTable    string           // the name of the join table, if specified explicitly via struct tag, otherwise blank
+	Model        *Model           // the primary Model which contains this relationship
+	FieldName    string           // the name of the struct field with this relationship
+	Idx          int              // the index of the struct field with this relationship
+	RelatedType  reflect.Type     // the reflect.Type for the struct field named by FieldName
+	RelatedModel *Model           // the Model representing the type of the embedded slice/struct
+	Kind         reflect.Kind     // field kind (struct, slice, ...)
+	m2mInverse   *Relationship    // the "inverse" side of an explicit ManyToMany relationship
+	Nullable     bool             // whether the foreign key for this relationship can have a nullable column
+	OriginalField reflect.StructField // the original reflect.StructField object associated with this relationship
+}
+
+// ComputeJoinTable - computes the name of the join table for ManyToMany relationships.
+// will return a snake_cased autogenerated name for unidirectional ManyToMany .
+// the "implicit" behavior is invoked upon one of the following conditions being met:
+// - the primary Relationship.Model
+func (r *Relationship) ComputeJoinTable() string {
+	if r.JoinTable != "" {
+		return r.JoinTable
+	}
+	otherSide := r.RelatedModel.TableName
+	if r.Model.embeddedIsh {
+		otherSide = pascalToSnakeCase(r.FieldName)
+	}
+	return r.Model.TableName + "_" + otherSide
+}
+
+func (r *Relationship) relatedID() *Field {
+	return r.RelatedModel.Fields[r.RelatedModel.IDField]
+}
+
+func (r *Relationship) primaryID() *Field {
+	return r.Model.Fields[r.Model.IDField]
+}
+
+func (r *Relationship) joinField() string {
+	if r.Type == ManyToOne {
+		return r.RelatedModel.Name + "ID"
+	}
+	if r.Type == ManyToMany && !r.Model.embeddedIsh {
+		return r.RelatedModel.Name + "ID"
+	}
+	return r.FieldName + "ID"
+}
+
+func (r *Relationship) m2mIsh() bool {
+	needsMany := false
+	if !r.Model.embeddedIsh && r.RelatedModel.embeddedIsh {
+		rr, ok := r.RelatedModel.Relationships[r.Model.Name]
+		if ok && rr.Type != ManyToOne {
+			needsMany = true
+		}
+	}
+	return ((r.Model.embeddedIsh && !r.RelatedModel.embeddedIsh) || needsMany) &&
+		r.Type == HasMany
+}
+
+func (r *Relationship) joinInsert(v reflect.Value, e *Query, pfk any) error {
+	if r.Type != ManyToMany &&
+		!r.m2mIsh() {
+		return nil
+	}
+	ichild := v
+	for ichild.Kind() == reflect.Ptr {
+		ichild = ichild.Elem()
+	}
+	if ichild.Kind() == reflect.Struct {
+		jtable := r.ComputeJoinTable()
+		jargs := make([]any, 0)
+		jcols := make([]string, 0)
+		jcols = append(jcols, fmt.Sprintf("%s_id",
+			r.Model.TableName,
+		))
+		jargs = append(jargs, pfk)
+
+		jcols = append(jcols, r.RelatedModel.TableName+"_id")
+		jargs = append(jargs, ichild.FieldByName(r.RelatedModel.IDField).Interface())
+		var ecnt int
+		e.tx.QueryRow(e.ctx,
+			fmt.Sprintf("SELECT count(*) from %s where %s = $1 and %s = $2", r.ComputeJoinTable(), jcols[0], jcols[1]), jargs...).Scan(&ecnt)
+		if ecnt > 0 {
+			return nil
+		}
+		jsql := fmt.Sprintf("INSERT INTO %s (%s) VALUES ($1, $2)", jtable, strings.Join(jcols, ", "))
+		e.engine.logQuery("insert/join", jsql, jargs)
+		if !e.engine.dryRun {
+			_ = e.tx.QueryRow(e.ctx, jsql, jargs...).Scan()
+		}
+	}
+	return nil
+}
+
+func (r *Relationship) joinDelete(pk, fk any, q *Query) error {
+	dq := sb.Delete(r.ComputeJoinTable()).Where(fmt.Sprintf("%s_id = ?", r.Model.TableName), pk)
+	if fk != nil {
+		dq = dq.Where(fmt.Sprintf("%s_id = ?", r.RelatedModel.TableName), fk)
+	}
+	ds, aa := dq.MustSQL()
+	q.engine.logQuery("delete/join", ds, aa)
+	if !q.engine.dryRun {
+		_, err := q.tx.Exec(q.ctx, ds, aa...)
+		return err
+	}
+	return nil
+}
+
+func parseRelationship(field reflect.StructField, modelMap map[string]*Model, outerType reflect.Type, idx int, settings map[string]string) *Relationship {
+	rel := &Relationship{
+		Model:         modelMap[outerType.Name()],
+		RelatedModel:  modelMap[field.Type.Name()],
+		RelatedType:   field.Type,
+		Idx:           idx,
+		Kind:          field.Type.Kind(),
+		FieldName:     field.Name,
+		OriginalField: field,
+	}
+	if rel.RelatedType.Kind() == reflect.Slice || rel.RelatedType.Kind() == reflect.Array {
+		rel.RelatedType = rel.RelatedType.Elem()
+	}
+	if rel.RelatedModel == nil {
+		if rel.RelatedType.Name() == "" {
+			rt := rel.RelatedType
+			for rt.Kind() == reflect.Ptr || rt.Kind() == reflect.Slice || rt.Kind() == reflect.Array {
+				rel.Nullable = true
+				rel.RelatedType = rel.RelatedType.Elem()
+				rt = rel.RelatedType
+			}
+		}
+		rel.RelatedModel = modelMap[rel.RelatedType.Name()]
+		if _, ok := modelMap[rel.RelatedType.Name()]; !ok {
+			rel.RelatedModel = parseModel(reflect.New(rel.RelatedType).Interface())
+			modelMap[rel.RelatedType.Name()] = rel.RelatedModel
+			parseModelFields(rel.RelatedModel, modelMap)
+			rel.RelatedModel.embeddedIsh = true
+		}
+	}
+	switch field.Type.Kind() {
+	case reflect.Struct:
+		rel.Type = HasOne
+	case reflect.Slice, reflect.Array:
+		rel.Type = HasMany
+	}
+	maybeM2m := settings["m2m"]
+	if maybeM2m == "" {
+		maybeM2m = settings["manytomany"]
+	}
+	if rel.Type == HasMany && maybeM2m != "" {
+		rel.JoinTable = maybeM2m
+	}
+	return rel
+}
+func addForeignKeyFields(ref *Relationship) {
+	if !ref.RelatedModel.embeddedIsh && !ref.Model.embeddedIsh {
+		ref.Type = BelongsTo
+	} else if !ref.Model.embeddedIsh && ref.RelatedModel.embeddedIsh {
+
+		if ref.Type == HasMany {
+			nr := &Relationship{
+				RelatedModel: ref.Model,
+				Model:        ref.RelatedModel,
+				Kind:         ref.RelatedModel.Type.Kind(),
+				Idx:          -1,
+				RelatedType:  ref.Model.Type,
+			}
+			nr.Type = ManyToOne
+			nr.FieldName = nr.RelatedModel.Name
+			ref.RelatedModel.Relationships[nr.FieldName] = nr
+		} else if ref.Type == HasOne {
+			ref.Type = BelongsTo
+		}
+	} else if ref.Model.embeddedIsh && !ref.RelatedModel.embeddedIsh {
+
+	}
+}

save_test.go

@@ -0,0 +1,114 @@
+package orm
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func save1(t assert.TestingT, e *Engine) {
+	u := author(t)
+	err := e.Model(&user{}).Create(&u)
+	assert.Nil(t, err)
+	f := friend(t)
+	err = e.Model(&user{}).Create(&f)
+	assert.Nil(t, err)
+	assert.NotZero(t, f.ID)
+	oldFavid := u.Favs.ID
+	u.Favs.Authors = append(u.Favs.Authors, f)
+	err = e.Model(&user{}).Save(&u)
+	assert.Nil(t, err)
+	assert.NotZero(t, u.Favs.ID)
+	assert.Equal(t, oldFavid, u.Favs.ID)
+}
+
+func save2(t assert.TestingT, e *Engine) {
+	insertBands(t, e)
+	u := author(t)
+	err := e.Model(&user{}).Create(&u)
+	assert.Nil(t, err)
+	assert.NotZero(t, u.Favs.ID)
+	s := iti_multi(u)
+	err = e.Model(&story{}).Save(s)
+	assert.Nil(t, err)
+	assert.NotZero(t, s.ID)
+	checkChapters(t, s)
+
+	s.Downloads = s.Downloads + 1
+	err = e.Model(&story{}).Save(s)
+	assert.Nil(t, err)
+	var ns story
+	err = e.Model(&story{}).Where("ID = ?", s.ID).Find(&ns)
+	assert.Nil(t, err)
+	assert.NotZero(t, ns.ID)
+	assert.NotZero(t, ns.Title)
+	assert.Equal(t, ns.Downloads, s.Downloads)
+}
+
+func updateRaw1(t assert.TestingT, e *Engine) {
+	insertBands(t, e)
+	u := author(t)
+	err := e.Model(&user{}).Create(&u)
+	assert.Nil(t, err)
+
+	s1 := iti_multi(u)
+	err = e.Model(&story{}).Save(s1)
+	assert.Nil(t, err)
+	assert.NotZero(t, s1.ID)
+	checkChapters(t, s1)
+
+	s2 := iti_single(u)
+	err = e.Model(&story{}).Save(s2)
+	assert.Nil(t, err)
+	assert.NotZero(t, s2.ID)
+	checkChapters(t, s2)
+
+	umap := make(map[string]any)
+	umap["Characters"] = `array_remove(characters, 'Brian Tatler')`
+	ra, err := e.Model(&chapter{}).WhereRaw("1 = ?", 1).UpdateRaw(umap)
+	assert.NotZero(t, ra)
+	assert.Nil(t, err)
+
+	stories := make([]story, 0)
+	err = e.Model(&story{}).Populate(PopulateAll, "Chapters.Bands").Find(&stories)
+	assert.Nil(t, err)
+	assert.NotEmpty(t, stories)
+	for _, ss := range stories {
+		checkChapters(t, &ss)
+		for _, c := range ss.Chapters {
+			steppedInShit := false
+			for _, b := range c.Characters {
+				if b == "Brian Tatler" {
+					steppedInShit = true
+					break
+				}
+			}
+			assert.False(t, steppedInShit)
+		}
+	}
+}
+
+func TestSave1(t *testing.T) {
+	e := initTest(t)
+	save1(t, e)
+	e.Disconnect()
+}
+
+func TestSave2(t *testing.T) {
+	e := initTest(t)
+	save2(t, e)
+	e.Disconnect()
+}
+
+func TestUpdateRaw1(t *testing.T) {
+	e := initTest(t)
+	updateRaw1(t, e)
+	e.Disconnect()
+}
+func BenchmarkSave(b *testing.B) {
+	b.Run("Save-1", bench(save1))
+	b.Run("Save-2", bench(save2))
+}
+
+func BenchmarkUpdateRaw(b *testing.B) {
+	bench(updateRaw1)(b)
+}

scan.go

@@ -0,0 +1,53 @@
+package orm
+
+import (
+	"github.com/jackc/pgx/v5"
+	"reflect"
+	"strings"
+)
+
+func buildScanDest(val reflect.Value, model *Model, fk *Relationship, cols []string, anonymousCols map[string]map[string]*Field, fkDest any) ([]any, error) {
+	var dest []any
+
+	for _, col := range cols {
+		bcol := col
+		if strings.Contains(bcol, ".") {
+			_, bcol, _ = strings.Cut(bcol, ".")
+		}
+		field := model.FieldsByColumnName[bcol]
+		if field != nil && !field.isAnonymous() {
+			dest = append(dest, val.FieldByName(field.Name).Addr().Interface())
+		}
+	}
+	for fn, a := range anonymousCols {
+		iv := val.FieldByName(fn)
+		for _, field := range a {
+			dest = append(dest, iv.FieldByName(field.Name).Addr().Interface())
+		}
+	}
+
+	if fk.Type != BelongsTo {
+		dest = append(dest, fkDest)
+	}
+
+	return dest, nil
+}
+func scanRow(row pgx.Row, cols []string, anonymousCols map[string][]string, destVal reflect.Value, m *Model) error {
+	var scanDest []any
+	for _, col := range cols {
+		f := m.FieldsByColumnName[col]
+		if f != nil && f.ColumnType != "" && !f.isAnonymous() {
+			scanDest = append(scanDest, destVal.FieldByIndex(f.Original.Index).Addr().Interface())
+		}
+	}
+	for kcol := range anonymousCols {
+		f := m.FieldsByColumnName[kcol]
+		if f != nil {
+			for _, ef := range f.embeddedFields {
+				scanDest = append(scanDest, destVal.FieldByIndex(f.Original.Index).FieldByName(ef.Name).Addr().Interface())
+			}
+		}
+	}
+
+	return row.Scan(scanDest...)
+}

testing.go

@@ -1,9 +1,11 @@
 package orm
 
 import (
-	"context"
 	"fmt"
 	"github.com/stretchr/testify/assert"
+	"math/rand/v2"
+	"os"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -12,43 +14,55 @@ import (
 )
 
 type chapter struct {
-	ID            bson.ObjectID `json:"_id"`
+	ChapterID     int64      `json:"chapterID" d:"pk:t;"`
 	Title         string     `json:"chapterTitle" form:"chapterTitle"`
-	ChapterID     int           `json:"chapterID" autoinc:"chapters"`
 	Index         int        `json:"index" form:"index"`
 	Words         int        `json:"words"`
 	Notes         string     `json:"notes" form:"notes"`
-	Genre         []string      `json:"genre" form:"genre"`
+	Genre         []string   `json:"genre" form:"genre" d:"type:text[]"`
 	Bands         []band     `json:"bands" ref:"band,bands"`
-	Characters    []string      `json:"characters" form:"characters"`
-	Relationships [][]string    `json:"relationships" form:"relationships"`
+	Characters    []string   `json:"characters" form:"characters" d:"type:text[]"`
+	Relationships [][]string `json:"relationships" form:"relationships" d:"type:jsonb"`
 	Adult         bool       `json:"adult" form:"adult"`
 	Summary       string     `json:"summary" form:"summary"`
 	Hidden        bool       `json:"hidden" form:"hidden"`
 	LoggedInOnly  bool       `json:"loggedInOnly" form:"loggedInOnly"`
 	Posted        time.Time  `json:"datePosted"`
-	FileName      string        `json:"fileName"`
-	Text          string        `json:"text" gridfs:"story_text,/stories/{{.ChapterID}}.txt"`
+	FileName      string     `json:"fileName" d:"-"`
+	Text          string     `json:"text" d:"column:content" gridfs:"story_text,/stories/{{.ChapterID}}.txt"`
 }
 
 type band struct {
-	ID         int64 `json:"_id"`
-	Document   `json:",inline" coll:"bands"`
+	Document   `json:",inline" d:"table:bands"`
+	ID         int64    `json:"_id" d:"pk;"`
 	Name       string   `json:"name" form:"name"`
 	Locked     bool     `json:"locked" form:"locked"`
-	Characters []string `json:"characters" form:"characters"`
+	Characters []string `json:"characters" form:"characters" d:"type:text[]"`
 }
 type user struct {
-	ID       int64 `json:"_id"`
-	Document `json:",inline" coll:"users"`
+	Document `json:",inline" d:"table:users"`
+	ID       int64  `json:"_id" d:"pk;"`
 	Username string `json:"username"`
-	Favs     []user `json:"favs" ref:"user"`
+	Favs     favs   `json:"favs" ref:"user"`
+	Roles    []role `d:"m2m:user_roles"`
+}
+
+type role struct {
+	ID    int64 `d:"pk"`
+	Name  string
+	Users []user `d:"m2m:user_roles"`
+}
+
+type favs struct {
+	ID      int64 `d:"pk"`
+	Stories []story
+	Authors []user
 }
 type story struct {
-	ID        int64 `json:"_id"`
-	Document  `json:",inline" coll:"stories"`
+	Document  `json:",inline" d:"table:stories"`
+	ID        int64     `json:"_id" d:"pk;"`
 	Title     string    `json:"title" form:"title"`
-	Author    *user     `json:"author" ref:"user"`
+	Author    user      `json:"author" ref:"user"`
 	CoAuthor  *user     `json:"coAuthor" ref:"user"`
 	Chapters  []chapter `json:"chapters"`
 	Recs      int       `json:"recs"`
@@ -64,86 +78,119 @@ type somethingWithNestedChapters struct {
 	NestedText string    `json:"text" gridfs:"nested_text,/nested/{{.ID}}.txt"`
 }
 
-func (s *somethingWithNestedChapters) Id() any {
-	return s.ID
+func isTestBench(t assert.TestingT) bool {
+	_, ok := t.(*testing.B)
+	return ok
 }
 
-func (s *somethingWithNestedChapters) SetId(id any) {
-	s.ID = id.(int64)
+func friend(t assert.TestingT) user {
+	ID := int64(83378)
+	if isTestBench(t) {
+		//ID = 0
+		//ID = rand.Int64N(100000) + 1
 	}
-
-func (s *story) Id() any {
-	return s.ID
-}
-
-func (s *band) Id() any {
-	return s.ID
-}
-func (s *user) Id() any {
-	return s.ID
-}
-
-func (s *story) SetId(id any) {
-	s.ID = id.(int64)
-	//var t IDocument =s
-}
-
-func (s *band) SetId(id any) {
-	s.ID = id.(int64)
-}
-
-func (s *user) SetId(id any) {
-	s.ID = id.(int64)
-}
-
-var author = user{
-	Username: "tablet.exe",
-	Favs: []user{
-		{
+	return user{
 		Username: "DarQuiel7",
-		},
-	},
+		ID:       ID,
+	}
+}
+func author(t assert.TestingT) user {
+	ID := int64(85783)
+	if isTestBench(t) {
+		//ID = 0
+	}
+	return user{
+		Username: "tablet.exe",
+		ID:       ID,
+	}
 }
 
-func genChaps(single bool) []chapter {
+func genChaps(single bool, aceil int) []chapter {
 	var ret []chapter
 	var ceil int
 	if single {
 		ceil = 1
 	} else {
-		ceil = 5
+		ceil = aceil
 	}
-	emptyRel := make([][]string, 0)
-	emptyRel = append(emptyRel, make([]string, 0))
-	relMap := [][][]string{
+
+	relMap := make([][][]string, 0)
+	bands := make([][]band, 0)
+	charMap := make([][]string, 0)
+	for i := range ceil {
+		curChars := make([]string, 0)
+		curBands := make([]band, 0)
+		curBands = append(curBands, diamondHead)
+		curChars = append(curChars, diamondHead.Characters...)
 		{
-			{"Sean Harris", "Brian Tatler"},
-		},
+			randMin := max(i+1, 1)
+			randMax := min(i+1, randMin+1)
+			mod1 := max(rand.IntN(randMin), 1)
+			mod2 := max(rand.IntN(randMax+1), 1)
+			if (mod1%mod2 == 0 || (mod1%mod2) == 2) && i > 0 {
+				curBands = append(curBands, bodom)
+				curChars = append(curChars, bodom.Characters...)
+			}
+		}
+		crel := make([][]string, 0)
+		numRels := rand.IntN(3)
+		seenRels := make(map[string]bool)
+		for len(crel) <= numRels {
+			arel := make([]string, 0)
+			randRelChars := rand.IntN(3)
+			numRelChars := 0
+			if randRelChars == 1 {
+				numRelChars = 3
+			} else if randRelChars == 2 {
+				numRelChars = 2
+			}
+			if numRelChars == 0 {
+				continue
+			}
+			seen := make(map[string]bool)
+			for len(arel) < numRelChars {
+				char := diamondHead.Characters[rand.IntN(len(diamondHead.Characters))]
+				if !seen[char] {
+					arel = append(arel, char)
+					seen[char] = true
+				}
+
+			}
+			slices.Sort(arel)
+			maybeSeen := strings.Join(arel, "/")
+			if maybeSeen != "" && !seenRels[maybeSeen] {
+				seenRels[maybeSeen] = true
+				crel = append(crel, arel)
+			}
+		}
 		{
-			{"Sean Harris", "Brian Tatler"},
-			{"Duncan Scott", "Colin Kimberley"},
-		},
-		{
-			{"Duncan Scott", "Colin Kimberley"},
-		},
-		emptyRel,
-		{
-			{"Sean Harris", "Colin Kimberley", "Brian Tatler"},
-		},
+			numChars := rand.IntN(len(curChars)-1) + 1
+			seen := make(map[string]bool)
+			cchars := make([]string, 0)
+			for len(cchars) <= numChars {
+				char := curChars[rand.IntN(len(curChars))]
+				if !seen[char] {
+					cchars = append(cchars, char)
+					seen[char] = true
+				}
+			}
+			charMap = append(charMap, cchars)
+		}
+		relMap = append(relMap, crel)
+		bands = append(bands, curBands)
 	}
 	l := loremipsum.New()
 
-	for i := 0; i < ceil; i++ {
+	for i := range ceil {
 		spf := fmt.Sprintf("%d.md", i+1)
-		ret = append(ret, chapter{
-			ID:            bson.NewObjectID(),
+		c := chapter{
 			Title:         fmt.Sprintf("-%d-", i+1),
 			Index:         i + 1,
 			Words:         50,
 			Notes:         "notenotenote !!!",
 			Genre:         []string{"Slash"},
-			Bands:         []band{diamondHead},
-			Characters:    []string{"Sean Harris", "Brian Tatler", "Duncan Scott", "Colin Kimberley"},
+			Bands:         bands[i],
+			Characters:    charMap[i],
 			Relationships: relMap[i],
 			Adult:         true,
 			Summary:       l.Paragraph(),
@@ -151,7 +198,10 @@ func genChaps(single bool) []chapter {
 			LoggedInOnly:  true,
 			FileName:      spf,
 			Text:          strings.Join(l.ParagraphList(10), "\n\n"),
-		})
+			Posted:        time.Now().Add(time.Hour * time.Duration(int64(24*7*i))),
+		}
+
+		ret = append(ret, c)
 	}
 
 	return ret
@@ -160,51 +210,35 @@ func genChaps(single bool) []chapter {
 func doSomethingWithNested() somethingWithNestedChapters {
 	l := loremipsum.New()
 	swnc := somethingWithNestedChapters{
-		Chapters:   genChaps(false),
+		Chapters:   genChaps(false, 7),
 		NestedText: strings.Join(l.ParagraphList(15), "\n\n"),
 	}
 	return swnc
 }
-func iti_single() story {
-	return story{
+func iti_single(a user) *story {
+	return &story{
 		Title:     "title",
 		Completed: true,
-		Chapters:  genChaps(true),
+		Author:    a,
+		Chapters:  genChaps(true, 1),
 	}
 }
 
-func iti_multi() story {
-	return story{
+func iti_multi(a user) *story {
+	return &story{
 		Title:     "Brian Tatler Fucked and Abused Sean Harris",
 		Completed: false,
-		Chapters:  genChaps(false),
+		Author:    a,
+		Chapters:  genChaps(false, 5),
 	}
 }
 
-func iti_blank() story {
-	t := iti_single()
+func iti_blank(a user) *story {
+	t := iti_single(a)
 	t.Chapters = make([]chapter, 0)
 	return t
 }
 
-func initTest() {
-	uri := "mongodb://127.0.0.1:27017"
-	db := "rockfic_ormTest"
-	ic, _ := mongo.Connect(options.Client().ApplyURI(uri))
-	ic.Database(db).Drop(context.TODO())
-	colls, _ := ic.Database(db).ListCollectionNames(context.TODO(), bson.M{})
-	if len(colls) < 1 {
-		mdb := ic.Database(db)
-		mdb.CreateCollection(context.TODO(), "bands")
-		mdb.CreateCollection(context.TODO(), "stories")
-		mdb.CreateCollection(context.TODO(), "users")
-	}
-	defer ic.Disconnect(context.TODO())
-	Connect(uri, db)
-	author.ID = 696969
-	ModelRegistry.Model(band{}, user{}, story{})
-}
-
 var metallica = band{
 	ID:   1,
 	Name: "Metallica",
@@ -243,12 +277,66 @@ var bodom = band{
 	},
 }
 
-func saveDoc(t *testing.T, doc IDocument) {
-	err := doc.Save()
+type commonTestFunc func(t assert.TestingT)
+
+var logTime = time.Now()
+
+func initCommonTest(t assert.TestingT) *Engine {
+	f, err := os.OpenFile(
+		fmt.Sprintf("test-logs/test-%d.log", logTime.UnixMilli()),
+		os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 	assert.Nil(t, err)
+	e, err := Open("postgres://testbed_user:123@localhost/real_test_db", &Config{
+		LogLevel: LevelQuery,
+		LogTo:    f,
+	})
+	assert.Nil(t, err)
+	e.Models(user{}, story{}, band{}, role{})
+	return e
 }
 
-func createAndSave(t *testing.T, doc IDocument) {
-	mdl := Create(doc).(IDocument)
-	saveDoc(t, mdl)
+func deleteAll(e *Engine) {
+	models := []any{&user{}, &story{}, &band{}, &role{}}
+	for _, model := range models {
+		e.Model(model).WhereRaw("true").Delete()
+	}
+}
+
+func initTest(t assert.TestingT) *Engine {
+	e := initCommonTest(t)
+	err := e.MigrateDropping()
+	assert.Nil(t, err)
+	return e
+}
+
+func insertBands(t assert.TestingT, e *Engine) {
+	toInsert := []*band{&bodom, &diamondHead}
+	/*if isTestBench(t) {
+		for i := range toInsert {
+			toInsert[i].ID = 0
+		}
+	}*/
+	for _, b := range toInsert {
+		err := e.Model(&band{}).Save(b)
+		assert.Nil(t, err)
+	}
+}
+
+func checkChapters(t assert.TestingT, s *story) {
+	for _, c := range s.Chapters {
+		assert.NotZero(t, c.ChapterID)
+		assert.NotZero(t, c.Text)
+		assert.NotZero(t, c.Posted)
+	}
+}
+
+func bench(fn func(assert.TestingT, *Engine)) func(b *testing.B) {
+	return func(b *testing.B) {
+		e := initCommonTest(b)
+		for b.Loop() {
+			deleteAll(e)
+			fn(b, e)
+		}
+		e.Disconnect()
+	}
 }

utils.go

@@ -0,0 +1,133 @@
+package orm
+
+import (
+	"fmt"
+	sb "github.com/henvic/pgq"
+	"reflect"
+	"regexp"
+	"strings"
+)
+
+var pascalRegex = regexp.MustCompile(`(?P<lowercase>[a-z])(?P<uppercase>[A-Z])`)
+var nonWordRegex = regexp.MustCompile(`[^a-zA-Z0-9_]`)
+
+func pascalToSnakeCase(str string) string {
+	step1 := pascalRegex.ReplaceAllString(str, `${lowercase}_${uppercase}`)
+	step2 := nonWordRegex.ReplaceAllString(step1, "_")
+	return strings.ToLower(step2)
+}
+
+func canConvertTo[T any](thisType reflect.Type) bool {
+	return thisType.ConvertibleTo(reflect.TypeFor[T]()) ||
+		thisType.ConvertibleTo(reflect.TypeFor[*T]()) ||
+		strings.TrimPrefix(thisType.Name(), "*") == strings.TrimPrefix(reflect.TypeFor[T]().Name(), "*")
+}
+
+func parseTags(t string) map[string]string {
+	tags := strings.Split(t, ";")
+	m := make(map[string]string)
+	for _, tag := range tags {
+		field := strings.Split(tag, ":")
+		if len(field) < 2 {
+			m[strings.ToLower(field[0])] = "t"
+		} else {
+			m[strings.ToLower(field[0])] = field[1]
+		}
+	}
+	return m
+}
+
+func capitalizeFirst(str string) string {
+	firstChar := strings.ToUpper(string([]byte{str[0]}))
+	return firstChar + string(str[1:])
+}
+
+func serialToRegular(str string) string {
+	return strings.ReplaceAll(strings.ToLower(str), "serial", "int")
+}
+func isZero(v reflect.Value) bool {
+	switch v.Kind() {
+	case reflect.String:
+		return v.String() == ""
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return v.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return v.Uint() == 0
+	case reflect.Bool:
+		return !v.Bool()
+	case reflect.Ptr, reflect.Interface:
+		return v.IsNil()
+	}
+	return v.IsZero()
+}
+
+func checkInsertable(v reflect.Value) {
+
+}
+
+func reflectSet(f reflect.Value, v any) {
+	if !f.CanSet() || v == nil {
+		return
+	}
+	switch f.Kind() {
+	case reflect.Int, reflect.Int64:
+		switch val := v.(type) {
+		case int64:
+			f.SetInt(val)
+		case int32:
+			f.SetInt(int64(val))
+		case int:
+			f.SetInt(int64(val))
+		case uint64:
+			f.SetInt(int64(val))
+		}
+	case reflect.String:
+		if s, ok := v.(string); ok {
+			f.SetString(s)
+		}
+	}
+}
+
+func logTrunc(length int, v []any) []any {
+	if length < 5 {
+		length = 5
+	}
+	trunced := make([]any, 0)
+	for _, it := range v {
+		if str, ok := it.(string); ok {
+			ntrunc := str[:min(length, len(str))]
+			if len(ntrunc) < len(str) {
+				ntrunc += "..."
+			}
+			trunced = append(trunced, ntrunc)
+		} else {
+			trunced = append(trunced, it)
+		}
+	}
+
+	return trunced
+}
+
+func isSliceOfStructs(rv reflect.Value) bool {
+	return rv.Kind() == reflect.Slice && rv.Type().Elem().Kind() == reflect.Struct
+}
+
+// MakePlaceholders - generates a string with `count`
+// occurences of a placeholder (`?`), delimited by a
+// comma and a space
+func MakePlaceholders(count int) string {
+	if count < 1 {
+		return ""
+	}
+	var ph []string
+	for range count {
+		ph = append(ph, "?")
+	}
+	return strings.Join(ph, ", ")
+}
+
+func wrapQueryIn(s sb.SelectBuilder, idName string) sb.SelectBuilder {
+	return s.Prefix(
+		fmt.Sprintf("%s in (",
+			idName)).Suffix(")")
+}