rework Populate api

document.go

@@ -3,7 +3,9 @@ package orm
 import (
 	"encoding/json"
 	"go.mongodb.org/mongo-driver/v2/bson"
+	"log"
 	"reflect"
+	"strings"
 	"time"
 )
 
@@ -15,6 +17,7 @@ type Document struct {
 	model           *Model          `bson:"-"`
 	exists          bool            `bson:"-"`
 	self            any             `bson:"-"`
+	raw             any             `bson:"-"`
 	populatedFields map[string]bool `bson:"-"`
 }
 
@@ -55,6 +58,7 @@ type IDocument interface {
 	Remove() error
 	Save() error
 	SaveWith(opts *SaveOptions) error
+	Populate(fields ...string)
 	SetSelf(arg interface{})
 	getExists() bool
 	setExists(n bool)
@@ -68,6 +72,8 @@ type IDocument interface {
 	markPopulated(field string)
 	markDepopulated(field string)
 	newPopulationMap()
+	getRaw() any
+	setRaw(raw any)
 }
 
 type SaveOptions struct {
@@ -274,3 +280,45 @@ func (d *Document) Swap(field string, i, j int) error {
 
 	return nil
 }
+
+func (d *Document) Populate(fields ...string) {
+	_, cm, _ := ModelRegistry.HasByName(d.model.typeName)
+
+	if cm != nil {
+		rawDoc := d.raw
+		for _, field := range fields {
+			// 0 = fieldname, 1 = typename, 2 = bson name
+
+			r, _ := readFields(field, cm)
+
+			if r.exists {
+				// get self
+				// get ptr
+				// find
+				// unmarshal...
+				htt := r.HydratedType
+				if htt.Kind() == reflect.Pointer || htt.Kind() == reflect.Slice {
+					htt = htt.Elem()
+				}
+				if strings.HasSuffix(field, ".") || strings.HasPrefix(field, ".") {
+					log.Printf("WARN: invalid field name passed to Populate(). skipping...\n")
+					continue
+				}
+
+				tto := r.HydratedType
+				if tto.Kind() == reflect.Pointer || tto.Kind() == reflect.Slice {
+					tto = tto.Elem()
+				}
+				_, refColl, _ := ModelRegistry.HasByName(tto.Name())
+				var tmp1 interface{}
+				asIDocument, docOk := d.self.(IDocument)
+				if docOk {
+					asIDocument.markPopulated(field)
+				}
+
+				tmp1 = populate(r, refColl.collection, rawDoc, field, reflect.ValueOf(d.self).Interface())
+				d.self = tmp1
+			}
+		}
+	}
+}

document_internals.go

@@ -9,6 +9,7 @@ import (
 	"go.mongodb.org/mongo-driver/v2/mongo"
 	"go.mongodb.org/mongo-driver/v2/mongo/options"
 	"reflect"
+	"strings"
 	"time"
 )
 
@@ -79,9 +80,11 @@ func serializeIDs(input interface{}, isJson bool, populated map[string]bool, par
 				}
 				var ip bool
 				for k1, v1 := range populated {
-					if k1 == parent {
+					if k1 == descent || k1 == parent {
 						ip = v1
 						break
+					} else if strings.HasPrefix(k1, descent) {
+						ip = v1
 					}
 				}
 				if terr == nil {
@@ -93,7 +96,9 @@ func serializeIDs(input interface{}, isJson bool, populated map[string]bool, par
 								rarr = append(rarr, getID(fv.Index(j).Interface()))
 							} else {
 								if ip {
-									ret0[bbson.Name] = serializeIDs(fv.Index(j).Interface(), isJson, populated, descent)
+									rarr = append(rarr, serializeIDs(fv.Index(j).Interface(), isJson, populated, descent))
+								} else {
+									rarr = append(rarr, getID(fv.Index(j).Interface()))
 								}
 							}
 						}
@@ -101,15 +106,15 @@ func serializeIDs(input interface{}, isJson bool, populated map[string]bool, par
 					} else if !ok {
 						panic(fmt.Sprintf("referenced model slice at '%s.%s' does not implement HasID", nameOf(input), ft.Name))
 					} else {
-						if reflect.ValueOf(ifc).IsNil() {
+						if reflect.ValueOf(ifc).IsNil() || reflect.ValueOf(ifc.Id()).IsZero() {
 							ret0[bbson.Name] = nil
 						} else {
 							if !isJson {
 								ret0[bbson.Name] = ifc.Id()
 							} else {
-								if ip {
+								if ip && bbson.Name != "-" {
 									ret0[bbson.Name] = serializeIDs(fv.Interface(), isJson, populated, descent)
-								} else {
+								} else if bbson.Name != "-" {
 									ret0[bbson.Name] = ifc.Id()
 								}
 							}
@@ -313,3 +318,11 @@ func (d *Document) newPopulationMap() {
 		d.populatedFields = make(map[string]bool)
 	}
 }
+
+func (d *Document) setRaw(raw any) {
+	d.raw = raw
+}
+
+func (d *Document) getRaw() any {
+	return d.raw
+}

model.go

@@ -115,9 +115,6 @@ func (m *Model) Find(query interface{}, opts *options.FindOptionsBuilder) (*Quer
 	q, err := m.FindRaw(query, opts)
 	//idoc := (*DocumentSlice)(qqv.Elem().UnsafePointer())
 	idoc := make(DocumentSlice, 0)
-	for i := 0; i < qqv.Elem().Len(); i++ {
-		idoc = append(idoc, qqv.Elem().Index(i).Interface().(IDocument))
-	}
 
 	if err == nil {
 		rawRes := bson.A{}
@@ -132,8 +129,13 @@ func (m *Model) Find(query interface{}, opts *options.FindOptionsBuilder) (*Quer
 			qq.reOrganize()
 			err = nil
 		}
-		for _, doc := range idoc {
+		for i := 0; i < qqv.Elem().Len(); i++ {
+			idoc = append(idoc, qqv.Elem().Index(i).Interface().(IDocument))
+		}
+		for i, doc := range idoc {
 			doc.setModel(*m)
+			doc.SetSelf(doc)
+			doc.setRaw(rawRes[i])
 		}
 	}
 
@@ -171,6 +173,7 @@ func (m *Model) FindOne(query interface{}, options *options.FindOneOptionsBuilde
 	}
 	qqn := ModelRegistry.new_(m.typeName)
 	idoc, ok := qqn.(IDocument)
+	idoc.setRaw(raw)
 
 	qq := &Query{
 		collection: m.getColl(),
@@ -188,6 +191,7 @@ func (m *Model) FindOne(query interface{}, options *options.FindOneOptionsBuilde
 	if ok {
 		idoc.setExists(true)
 		idoc.setModel(*m)
+		idoc.setRaw(raw)
 	}
 	idoc.SetSelf(idoc)
 	return qq, err

query.go

@@ -8,7 +8,6 @@ import (
 	"github.com/fatih/structtag"
 	"go.mongodb.org/mongo-driver/v2/bson"
 	"go.mongodb.org/mongo-driver/v2/mongo"
-	"log"
 	"reflect"
 	"strings"
 )
@@ -36,6 +35,9 @@ func populate(r Reference, rcoll string, rawDoc interface{}, d string, src inter
 	if srt.Kind() == reflect.Pointer {
 		srt = rt.Elem()
 	}
+	if rv.Kind() == reflect.Pointer && reflect.ValueOf(src).IsNil() {
+		return src
+	}
 	if rv.Kind() != reflect.Pointer {
 		rv = reflect.New(rt)
 		rv.Elem().Set(reflect.ValueOf(src))
@@ -45,6 +47,7 @@ func populate(r Reference, rcoll string, rawDoc interface{}, d string, src inter
 	type bsonWhat struct {
 		What string
 	}
+	var isNotStructOrSlice bool
 	var w bsonWhat
 	switch rawDoc.(type) {
 	case bson.A:
@@ -76,6 +79,9 @@ func populate(r Reference, rcoll string, rawDoc interface{}, d string, src inter
 		for i, el := range rawDoc.(bson.A) {
 			it := rahh.Index(i)
 			popped := populate(r, rcoll, el, next, it.Interface())
+			if pidoc, pok := popped.(IDocument); pok {
+				pidoc.setRaw(el)
+			}
 			poppedVal := reflect.ValueOf(popped)
 			if poppedVal.Kind() == reflect.Pointer {
 				rvs = reflect.Append(rvs, poppedVal.Elem())
@@ -136,6 +142,14 @@ func populate(r Reference, rcoll string, rawDoc interface{}, d string, src inter
 			}
 		}
 		intermediate := populate(r, rcoll, dd[fieldsMap[2]], next, sf.Interface())
+		/*if iidoc, idocOk := intermediate.(IDocument); idocOk {
+			if (reflect.ValueOf(intermediate).CanAddr() && !reflect.ValueOf(intermediate).IsNil()) || !reflect.ValueOf(intermediate).IsZero() {
+				iiid, iok := intermediate.(HasID)
+				if intermediate != nil && iok && !reflect.ValueOf(iiid.Id()).IsZero() {
+					iidoc.setRaw(dd[fieldsMap[2]])
+				}
+			}
+		}*/
 		if rsf.CanSet() {
 			ival := reflect.ValueOf(intermediate)
 			if ival.Kind() != reflect.Pointer && rsf.Kind() == reflect.Pointer {
@@ -149,6 +163,7 @@ func populate(r Reference, rcoll string, rawDoc interface{}, d string, src inter
 			src = intermediate
 		}
 	default:
+		isNotStructOrSlice = true
 		tto := r.HydratedType
 		if tto.Kind() == reflect.Pointer || tto.Kind() == reflect.Slice {
 			tto = tto.Elem()
@@ -159,7 +174,10 @@ func populate(r Reference, rcoll string, rawDoc interface{}, d string, src inter
 		q := bson.M{"_id": rawDoc}
 		reso := DB.Collection(rcoll).FindOne(context.TODO(), q)
 		if !errors.Is(reso.Err(), mongo.ErrNoDocuments) {
-			reso.Decode(t)
+			var anotherMap = make(bson.M)
+			reso.Decode(&anotherMap)
+			reflect.ValueOf(t).Elem().Set(reflect.ValueOf(rerere(anotherMap, tto)).Elem())
+			t.setRaw(anotherMap)
 		}
 		hatred := rv
 		if hatred.Kind() == reflect.Pointer {
@@ -169,7 +187,12 @@ func populate(r Reference, rcoll string, rawDoc interface{}, d string, src inter
 			if reflect.ValueOf(t).Kind() == reflect.Pointer {
 				fmt.Println(reflect.ValueOf(t).Elem().Type().String())
 				fmt.Println(rv.Type().String())
-				hatred.Set(reflect.ValueOf(t).Elem())
+				if hatred.Kind() == reflect.Pointer {
+					hatred.Set(reflect.ValueOf(t))
+				} else {
+					hatred.Set(reflect.ValueOf(t).Elem())
+				}
+				recover()
 			} else {
 				hatred.Set(reflect.ValueOf(t))
 			}
@@ -185,12 +208,18 @@ func populate(r Reference, rcoll string, rawDoc interface{}, d string, src inter
 		sidoc, sok := rv.Interface().(IDocument)
 		if sok {
 			sidoc.SetSelf(rv.Interface())
+			if !isNotStructOrSlice {
+				sidoc.setRaw(rawDoc)
+			}
 		}
 		return rv.Interface()
 	}
 	sidoc, sok := src.(IDocument)
 	if sok {
 		sidoc.SetSelf(src)
+		if !isNotStructOrSlice {
+			sidoc.setRaw(rawDoc)
+		}
 	}
 	return src
 }
@@ -227,71 +256,76 @@ func (q *Query) LoadFile(fields ...string) *Query {
 	return q
 }
 
-// Populate populates document references via reflection
+func readFields(field string, m *Model) (Reference, string) {
-func (q *Query) Populate(fields ...string) *Query {
+	var r Reference
-	_, cm, _ := ModelRegistry.HasByName(q.model.typeName)
+	var keptKey string
-
+	if m == nil {
-	if cm != nil {
+		return r, keptKey
-		rawDoc := q.rawDoc
+	}
-		for _, field := range fields {
+	for k2, v := range m.references {
-			// 0 = fieldname, 1 = typename, 2 = bson name
+		if strings.HasPrefix(k2, field) {
-
+			r = v
-			var r Reference
+			keptKey = k2
-
+			break
-			for k2, v := range cm.references {
+		} else if _, ok := m.references[field]; !ok {
-				if strings.HasPrefix(k2, field) {
+			splitSegs := strings.Split(field, ".")
-					r = v
+			for ii := 0; ii < len(splitSegs)-1; ii++ {
-					break
+				mr, ok2 := m.references[splitSegs[ii]]
-				}
+				if ok2 {
-			}
+					_, sec, _ := ModelRegistry.HasByName(mr.Model)
-
+					if sec != nil {
-			if r.exists {
+						refff, ok3 := sec.references[splitSegs[ii+1]]
-				// get self
+						if ok3 {
-				// get ptr
+							r = refff
-				// find
+							keptKey = k2
-				// unmarshal...
+							break
-				htt := r.HydratedType
-				if htt.Kind() == reflect.Pointer || htt.Kind() == reflect.Slice {
-					htt = htt.Elem()
-				}
-				if strings.HasSuffix(field, ".") || strings.HasPrefix(field, ".") {
-					log.Printf("WARN: invalid field name passed to Populate(). skipping...\n")
-					continue
-				}
-
-				tto := r.HydratedType
-				if tto.Kind() == reflect.Pointer || tto.Kind() == reflect.Slice {
-					tto = tto.Elem()
-				}
-				_, refColl, _ := ModelRegistry.HasByName(tto.Name())
-				var tmp1 interface{}
-				if arr, ok := rawDoc.(bson.A); ok {
-					typ := reflect.PointerTo(cm.Type)
-					slic := reflect.New(reflect.SliceOf(typ))
-					for i, val2 := range arr {
-						ref := reflect.ValueOf(q.doc)
-						if ref.Kind() == reflect.Pointer {
-							ref = ref.Elem()
-						}
-						src := ref.Index(i).Interface()
-						inter := populate(r, refColl.collection, val2, field, src)
-						if reflect.ValueOf(inter).Kind() == reflect.Pointer {
-							slic.Elem().Set(reflect.Append(slic.Elem(), reflect.ValueOf(inter)))
 						} else {
-							slic.Elem().Set(reflect.Append(slic, reflect.ValueOf(inter)))
+							joined := strings.Join(splitSegs[ii+1:], ".")
+							inner1, innerKey := readFields(joined, sec)
+							if inner1.exists {
+								r = inner1
+								keptKey = innerKey
+							}
+							break
 						}
 					}
-					tmp1 = slic.Interface()
-				} else {
-					asIDocument, docOk := q.doc.(IDocument)
-					if docOk {
-						asIDocument.markPopulated(field)
-					}
-					tmp1 = populate(r, refColl.collection, rawDoc, field, reflect.ValueOf(q.doc).Interface())
 				}
-				q.doc = tmp1
 			}
 		}
+		if keptKey != "" {
+			break
+		}
+	}
+	return r, keptKey
+}
+
+// Populate populates document references via reflection
+func (q *Query) Populate(fields ...string) *Query {
+	vvv := reflect.ValueOf(q.doc)
+	if vvv.Kind() == reflect.Pointer {
+		vvv = vvv.Elem()
+	}
+	if vvv.Kind() == reflect.Slice {
+		typ := reflect.PointerTo(q.model.Type)
+		slic := reflect.New(reflect.SliceOf(typ))
+		for i := 0; i < vvv.Len(); i++ {
+			val2 := vvv.Index(i).Interface()
+			aid, docOk := val2.(IDocument)
+			if docOk {
+				rdoc := q.rawDoc.(bson.A)
+				aid.setRaw(rdoc[i])
+				aid.Populate(fields...)
+				if reflect.ValueOf(aid).Kind() == reflect.Pointer {
+					slic.Elem().Set(reflect.Append(slic.Elem(), reflect.ValueOf(aid)))
+				} else {
+					slic.Elem().Set(reflect.Append(slic, reflect.ValueOf(aid)))
+				}
+			}
+		}
+		q.doc = slic.Interface()
+	} else if asDoc, ok2 := q.doc.(IDocument); ok2 {
+		asDoc.setRaw(q.rawDoc)
+		asDoc.Populate(fields...)
 	}
 	return q
 }
@@ -347,7 +381,14 @@ func rerere(input interface{}, resType reflect.Type) interface{} {
 	if resType.Kind() == reflect.Pointer {
 		resType = resType.Elem()
 	}
-	resV := reflect.New(resType)
+	var resV reflect.Value
+	newInstance := ModelRegistry.newForType(resType)
+	if newInstance == nil {
+		resV = reflect.New(resType)
+	} else {
+		resV = reflect.ValueOf(newInstance)
+	}
+
 	var rve = resV
 	if rve.Kind() == reflect.Pointer {
 		rve = resV.Elem()
@@ -398,8 +439,6 @@ func rerere(input interface{}, resType reflect.Type) interface{} {
 							fuck = fuck.Elem()
 						}
 						fv.Set(fuck)
-					} else {
-						fv.Set(reflect.Zero(ft.Type))
 					}
 					shouldBreak = true
 				} else {
@@ -414,8 +453,6 @@ func rerere(input interface{}, resType reflect.Type) interface{} {
 						} else {
 							fv.Set(reflect.ValueOf(tmp))
 						}
-					} else {
-						fv.Set(reflect.Zero(ft.Type))
 					}
 				}
 			}

registry.go

@@ -167,7 +167,7 @@ func parseTags(t reflect.Type, v reflect.Value, lastParsed string, eqCount int)
 					break
 				}
 				if refTag, ok := tags.Get("ref"); ok == nil {
-					sname := sft.Name + "@" + refTag.Name
+					sname := sft.Name
 					refs[sname] = makeRef(i, refTag.Name, sft.Name, sft.Type)
 				}
 				if gtag, ok := tags.Get("gridfs"); ok == nil {
@@ -244,6 +244,10 @@ func (r TModelRegistry) new_(n string) interface{} {
 	return nil
 }
 
+func (r TModelRegistry) newForType(rt reflect.Type) interface{} {
+	return r.new_(rt.Name())
+}
+
 func (r TModelRegistry) Get(name string) *Model {
 	model, ok := r[name]
 	if !ok {