feat: 完善扫描器、exec 及 schema 相关功能

- exec/scanner: 用 *interface{} 替换 **json.RawMessage 扫描目标，兼容 DuckDB 返回 map[string]interface{} 的场景；新增 toJSONRawMessage 转换函数
- exec/scanner: ScanVal 支持结构体指针，通过 JSON 中间层转换（DuckDB STRUCT 列）
- exec/scanner: 将 *sql.RawBytes 和 *[]byte 的处理从 ScanValContext 移入 scanner.ScanVal
- exec/query_executor: 简化 ScanValContext，移除私有 scan 方法
- exec: 补充 scanner 级别 ScanVal 测试用例
- internal/util/reflect: 重写 SafeSetVarValue，修复非指针 src 及 nil 指针字段的 panic
- internal/util/column_map: 恢复非匿名带标签结构体字段的展开逻辑
- schema: 新增 vector 列类型支持
- engine: 补充 DuckDB 相关配置
- dialect/sqlite3/vtab: 完善虚拟表适配器
- 各方言测试改用 sqlmock 虚拟连接

exec/query_executor.go

@@ -227,8 +227,8 @@ func (q QueryExecutor) ScanValContext(ctx context.Context, i interface{}) (bool,
 	if util.IsSlice(val.Kind()) {
 		switch i.(type) {
 		case *gsql.RawBytes: // do nothing
-		case *[]byte: // do nothing
-		case gsql.Scanner: // do nothing
+		case *[]byte:        // do nothing
+		case gsql.Scanner:   // do nothing
 		default:
 			return false, errScanValNonSlice
 		}
@@ -238,18 +238,14 @@ func (q QueryExecutor) ScanValContext(ctx context.Context, i interface{}) (bool,
 	if err != nil {
 		return false, err
 	}
-
 	defer func() { _ = scanner.Close() }()
 
 	if scanner.Next() {
-		err = scanner.ScanVal(i)
-		if err != nil {
+		if err = scanner.ScanVal(i); err != nil {
 			return false, err
 		}
-
 		return true, scanner.Err()
 	}
-
 	return false, scanner.Err()
 }
 

exec/query_executor_internal_test.go

@@ -3,6 +3,7 @@ package exec
 import (
 	"context"
 	"database/sql"
+	"database/sql/driver"
 	"encoding/json"
 	"fmt"
 	"strings"
@@ -13,6 +14,11 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
+// anyValueConverter 允许任意类型作为 driver.Value 透传，模拟 DuckDB 等返回 map/struct 的驱动
+type anyValueConverter struct{}
+
+func (anyValueConverter) ConvertValue(v interface{}) (driver.Value, error) { return v, nil }
+
 var (
 	testAddr1                  = "111 Test Addr"
 	testAddr2                  = "211 Test Addr"
@@ -929,9 +935,11 @@ func (qes *queryExecutorSuite) TestScanStruct() {
 	qes.EqualError(err, "queryExecutor error")
 	qes.False(found)
 
+	// NULL 值扫描进 string 字段：通过 **string 中间层正常处理，结果为空字符串
 	found, err = e.ScanStruct(&item)
-	qes.Error(err)
-	qes.False(found)
+	qes.NoError(err)
+	qes.True(found)
+	qes.Equal(StructWithTags{Address: "", Name: ""}, item)
 
 	found, err = e.ScanStruct(&item)
 	qes.NoError(err)
@@ -1242,6 +1250,182 @@ func (qes *queryExecutorSuite) TestScanVal_withValuerSlice() {
 	qes.Equal(JSONBoolArray{true, false, true}, bools)
 }
 
+func (qes *queryExecutorSuite) TestScanVal_withByteSlice_notFound() {
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "name" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"name"}))
+
+	e := newQueryExecutor(db, nil, `SELECT "name" FROM "items"`)
+	var b []byte
+	found, err := e.ScanVal(&b)
+	qes.NoError(err)
+	qes.False(found)
+	qes.Nil(b)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withByteSlice_queryError() {
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "name" FROM "items"`).
+		WillReturnError(fmt.Errorf("query error"))
+
+	e := newQueryExecutor(db, nil, `SELECT "name" FROM "items"`)
+	var b []byte
+	found, err := e.ScanVal(&b)
+	qes.EqualError(err, "query error")
+	qes.False(found)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withByteSlice_complexJSON() {
+	// 模拟 DuckDB 等驱动将 JSON 列直接以 map 形式返回
+	// 必须使用 mock.NewRows 而非 sqlmock.NewRows，才会使用自定义 converter
+	db, mock, err := sqlmock.New(sqlmock.ValueConverterOption(anyValueConverter{}))
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "output" FROM "items"`).
+		WillReturnRows(mock.NewRows([]string{"output"}).
+			AddRow(map[string]interface{}{"key": "value", "num": float64(42)}))
+
+	e := newQueryExecutor(db, nil, `SELECT "output" FROM "items"`)
+	var b []byte
+	found, err := e.ScanVal(&b)
+	qes.NoError(err)
+	qes.True(found)
+
+	var result map[string]interface{}
+	qes.NoError(json.Unmarshal(b, &result))
+	qes.Equal("value", result["key"])
+	qes.Equal(float64(42), result["num"])
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withRawBytes_notFound() {
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "name" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"name"}))
+
+	e := newQueryExecutor(db, nil, `SELECT "name" FROM "items"`)
+	var rb sql.RawBytes
+	found, err := e.ScanVal(&rb)
+	qes.NoError(err)
+	qes.False(found)
+	qes.Nil(rb)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withRawBytes_queryError() {
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "name" FROM "items"`).
+		WillReturnError(fmt.Errorf("query error"))
+
+	e := newQueryExecutor(db, nil, `SELECT "name" FROM "items"`)
+	var rb sql.RawBytes
+	found, err := e.ScanVal(&rb)
+	qes.EqualError(err, "query error")
+	qes.False(found)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withRawBytes_binaryDriver() {
+	// 驱动直接返回 []byte（如 BLOB 列），结果需独立拷贝
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	content := []byte("binary \x00 content")
+	mock.ExpectQuery(`SELECT "data" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"data"}).AddRow(content))
+
+	e := newQueryExecutor(db, nil, `SELECT "data" FROM "items"`)
+	var rb sql.RawBytes
+	found, err := e.ScanVal(&rb)
+	qes.NoError(err)
+	qes.True(found)
+	qes.Equal(sql.RawBytes(content), rb)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withStruct() {
+	// 模拟 DuckDB 将 JSON/STRUCT 列以 map[string]interface{} 返回
+	type DocItem struct {
+		Title string `json:"title"`
+		Score int    `json:"score"`
+	}
+	db, mock, err := sqlmock.New(sqlmock.ValueConverterOption(anyValueConverter{}))
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "doc" FROM "items"`).
+		WillReturnRows(mock.NewRows([]string{"doc"}).
+			AddRow(map[string]interface{}{"title": "hello", "score": float64(99)}))
+
+	e := newQueryExecutor(db, nil, `SELECT "doc" FROM "items"`)
+	var doc DocItem
+	found, err := e.ScanVal(&doc)
+	qes.NoError(err)
+	qes.True(found)
+	qes.Equal(DocItem{Title: "hello", Score: 99}, doc)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withStruct_null() {
+	// 驱动返回 NULL，结构体保持零值
+	type DocItem struct {
+		Title string `json:"title"`
+	}
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "doc" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"doc"}).AddRow(nil))
+
+	e := newQueryExecutor(db, nil, `SELECT "doc" FROM "items"`)
+	var doc DocItem
+	found, err := e.ScanVal(&doc)
+	qes.NoError(err)
+	qes.True(found)
+	qes.Equal(DocItem{}, doc)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withStruct_notFound() {
+	type DocItem struct {
+		Title string `json:"title"`
+	}
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "doc" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"doc"}))
+
+	e := newQueryExecutor(db, nil, `SELECT "doc" FROM "items"`)
+	var doc DocItem
+	found, err := e.ScanVal(&doc)
+	qes.NoError(err)
+	qes.False(found)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withStruct_queryError() {
+	type DocItem struct {
+		Title string `json:"title"`
+	}
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "doc" FROM "items"`).
+		WillReturnError(fmt.Errorf("query error"))
+
+	e := newQueryExecutor(db, nil, `SELECT "doc" FROM "items"`)
+	var doc DocItem
+	found, err := e.ScanVal(&doc)
+	qes.EqualError(err, "query error")
+	qes.False(found)
+}
+
+func (qes *queryExecutorSuite) TestScanVal_withStruct_sqlScanner() {
+	// 实现了 sql.Scanner 的结构体走原有直接扫描路径
+	db, mock, err := sqlmock.New()
+	qes.NoError(err)
+	mock.ExpectQuery(`SELECT "name" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("hello"))
+
+	e := newQueryExecutor(db, nil, `SELECT "name" FROM "items"`)
+	var ns sql.NullString
+	found, err := e.ScanVal(&ns)
+	qes.NoError(err)
+	qes.True(found)
+	qes.Equal(sql.NullString{String: "hello", Valid: true}, ns)
+}
+
 func TestQueryExecutorSuite(t *testing.T) {
 	suite.Run(t, new(queryExecutorSuite))
 }

exec/scanner.go

@@ -147,9 +147,9 @@ func (s *scanner) ScanStruct(i interface{}) error {
 
 		// 补全未知字段类型
 		if len(cols) != len(cm) {
-			colTypes, err := s.rows.ColumnTypes()
-			if err != nil {
-				return err
+			colTypes, ctErr := s.rows.ColumnTypes()
+			if ctErr != nil {
+				return ctErr
 			}
 			for _, t := range colTypes {
 				if _, ok := cm[t.Name()]; !ok {
@@ -166,7 +166,6 @@ func (s *scanner) ScanStruct(i interface{}) error {
 	}
 
 	scans, err := createColumnScans(s.columns, s.columnMap)
-
 	if err != nil {
 		return err
 	}
@@ -177,7 +176,12 @@ func (s *scanner) ScanStruct(i interface{}) error {
 
 	record := map[string]interface{}{}
 	for index, col := range s.columns {
-		record[col] = scans[index]
+		if pi, ok := scans[index].(*interface{}); ok {
+			raw := toJSONRawMessage(*pi)
+			record[col] = &raw
+		} else {
+			record[col] = scans[index]
+		}
 	}
 
 	util.AssignStructVals(i, record, s.columnMap)
@@ -198,10 +202,59 @@ func (s *scanner) ScanStructs(i interface{}) error {
 
 // ScanVal will scan the current row and column into i.
 func (s *scanner) ScanVal(i interface{}) error {
-	if err := s.rows.Scan(i); err != nil {
-		return err
+	switch v := i.(type) {
+	case *sql.RawBytes:
+		// 零拷贝扫描，rows.Close 前立即拷贝防止驱动回收缓冲区
+		if err := s.rows.Scan(v); err != nil {
+			return err
+		}
+		buf := make(sql.RawBytes, len(*v))
+		copy(buf, *v)
+		*v = buf
+	case *[]byte:
+		// 先扫描到 interface{}，驱动可能返回 []byte/string/map 等任意类型
+		var raw interface{}
+		if err := s.rows.Scan(&raw); err != nil {
+			return err
+		}
+		switch rv := raw.(type) {
+		case []byte:
+			*v = append([]byte(nil), rv...)
+		case sql.RawBytes:
+			*v = append([]byte(nil), []byte(rv)...)
+		case string:
+			*v = []byte(rv)
+		default:
+			if raw != nil {
+				var err error
+				*v, err = json.Marshal(raw)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	default:
+		// 指针-结构体且未实现 sql.Scanner：通过 JSON 中间层转换
+		if rv := reflect.ValueOf(i); rv.Kind() == reflect.Ptr && rv.Elem().Kind() == reflect.Struct {
+			if _, ok := i.(sql.Scanner); !ok {
+				var raw interface{}
+				if err := s.rows.Scan(&raw); err != nil {
+					return err
+				}
+				if raw == nil {
+					return s.Err()
+				}
+				data, err := json.Marshal(raw)
+				if err != nil {
+					return err
+				}
+				return json.Unmarshal(data, i)
+			}
+		}
+		if err := s.rows.Scan(i); err != nil {
+			return err
+		}
 	}
-
 	return s.Err()
 }
 
@@ -260,6 +313,23 @@ func checkScanValsTarget(i interface{}) (reflect.Value, error) {
 	return val, nil
 }
 
+func toJSONRawMessage(v interface{}) *json.RawMessage {
+	if v == nil {
+		return nil
+	}
+	var raw json.RawMessage
+	switch s := v.(type) {
+	case []byte:
+		raw = append(json.RawMessage(nil), s...)
+	case string:
+		raw = json.RawMessage(s)
+	default:
+		b, _ := json.Marshal(v)
+		raw = b
+	}
+	return &raw
+}
+
 func createColumnScans(cols []string, cm util.ColumnMap) (scans []interface{}, err error) {
 	scans = make([]interface{}, 0, len(cols))
 
@@ -278,7 +348,8 @@ func createColumnScans(cols []string, cm util.ColumnMap) (scans []interface{}, e
 			reflect.Bool:
 			scans = append(scans, reflect.New(reflect.PointerTo(data.GoType)).Interface())
 		case reflect.Map, reflect.Slice, reflect.Struct:
-			scans = append(scans, reflect.New(reflect.PointerTo(reflect.TypeOf(json.RawMessage{}))).Interface())
+			// 使用 *interface{} 接受任意驱动值（兼容 DuckDB 返回 map[string]interface{}）
+			scans = append(scans, new(interface{}))
 		default:
 			scans = append(scans, reflect.New(data.GoType).Interface())
 		}

exec/scanner_internal_test.go

@@ -1,9 +1,10 @@
 package exec
 
 import (
+	"database/sql"
+	"encoding/json"
 	"testing"
 
-	"git.fsdpf.net/go/db/exp"
 	"github.com/DATA-DOG/go-sqlmock"
 	"github.com/stretchr/testify/suite"
 )
@@ -81,14 +82,162 @@ func (s *scannerSuite) TestGetRecords() {
 				AddRow("111 Test Addr", "Test1"),
 		)
 
-	rows, err := db.Query("SELECT \\* FROM `items`")
+	rows, err := db.Query("SELECT * FROM `items`")
 	s.Require().NoError(err)
 
 	result, err := NewScanner(rows).GetRecords()
 	s.Require().NoError(err)
 
-	s.Equal([]exp.Record{
+	s.Equal([]map[string]any{
 		{"address": "111 Test Addr", "name": "Test1"},
 		{"address": "111 Test Addr", "name": "Test1"},
 	}, result)
 }
+
+func (s *scannerSuite) TestScanVal() {
+	db, mock, err := sqlmock.New()
+	s.Require().NoError(err)
+
+	mock.ExpectQuery(`SELECT "id" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(42))
+
+	rows, err := db.Query(`SELECT "id" FROM "items"`)
+	s.Require().NoError(err)
+
+	sc := NewScanner(rows)
+	s.True(sc.Next())
+
+	var id int64
+	err = sc.ScanVal(&id)
+	s.Require().NoError(err)
+	s.Equal(int64(42), id)
+}
+
+func (s *scannerSuite) TestScanVal_withRawBytes() {
+	db, mock, err := sqlmock.New()
+	s.Require().NoError(err)
+
+	mock.ExpectQuery(`SELECT "data" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"data"}).AddRow([]byte(testByteSliceContent)))
+
+	rows, err := db.Query(`SELECT "data" FROM "items"`)
+	s.Require().NoError(err)
+
+	sc := NewScanner(rows)
+	s.True(sc.Next())
+
+	var rb sql.RawBytes
+	err = sc.ScanVal(&rb)
+	s.Require().NoError(err)
+
+	_ = sc.Close()
+	// 关闭后缓冲区应已拷贝，值仍然有效
+	s.Equal(sql.RawBytes(testByteSliceContent), rb)
+}
+
+func (s *scannerSuite) TestScanVal_withByteSlice() {
+	db, mock, err := sqlmock.New()
+	s.Require().NoError(err)
+
+	mock.ExpectQuery(`SELECT "data" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"data"}).AddRow(testByteSliceContent))
+
+	rows, err := db.Query(`SELECT "data" FROM "items"`)
+	s.Require().NoError(err)
+
+	sc := NewScanner(rows)
+	s.True(sc.Next())
+
+	var b []byte
+	err = sc.ScanVal(&b)
+	s.Require().NoError(err)
+	s.Equal([]byte(testByteSliceContent), b)
+}
+
+func (s *scannerSuite) TestScanVal_withByteSlice_complexJSON() {
+	db, mock, err := sqlmock.New(sqlmock.ValueConverterOption(anyValueConverter{}))
+	s.Require().NoError(err)
+
+	payload := map[string]interface{}{"key": "val", "num": float64(1)}
+	mock.ExpectQuery(`SELECT "data" FROM "items"`).
+		WillReturnRows(mock.NewRows([]string{"data"}).AddRow(payload))
+
+	rows, err := db.Query(`SELECT "data" FROM "items"`)
+	s.Require().NoError(err)
+
+	sc := NewScanner(rows)
+	s.True(sc.Next())
+
+	var b []byte
+	err = sc.ScanVal(&b)
+	s.Require().NoError(err)
+
+	var got map[string]interface{}
+	s.Require().NoError(json.Unmarshal(b, &got))
+	s.Equal(payload, got)
+}
+
+func (s *scannerSuite) TestScanVal_withStruct() {
+	type DocItem struct {
+		Title string `json:"title"`
+		Score int    `json:"score"`
+	}
+	db, mock, err := sqlmock.New(sqlmock.ValueConverterOption(anyValueConverter{}))
+	s.Require().NoError(err)
+
+	mock.ExpectQuery(`SELECT "doc" FROM "items"`).
+		WillReturnRows(mock.NewRows([]string{"doc"}).
+			AddRow(map[string]interface{}{"title": "hello", "score": float64(99)}))
+
+	rows, err := db.Query(`SELECT "doc" FROM "items"`)
+	s.Require().NoError(err)
+
+	sc := NewScanner(rows)
+	s.True(sc.Next())
+
+	var doc DocItem
+	err = sc.ScanVal(&doc)
+	s.Require().NoError(err)
+	s.Equal(DocItem{Title: "hello", Score: 99}, doc)
+}
+
+func (s *scannerSuite) TestScanVal_withStruct_null() {
+	type DocItem struct {
+		Title string `json:"title"`
+	}
+	db, mock, err := sqlmock.New()
+	s.Require().NoError(err)
+
+	mock.ExpectQuery(`SELECT "doc" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"doc"}).AddRow(nil))
+
+	rows, err := db.Query(`SELECT "doc" FROM "items"`)
+	s.Require().NoError(err)
+
+	sc := NewScanner(rows)
+	s.True(sc.Next())
+
+	var doc DocItem
+	err = sc.ScanVal(&doc)
+	s.Require().NoError(err)
+	s.Equal(DocItem{}, doc)
+}
+
+func (s *scannerSuite) TestScanVal_withStruct_sqlScanner() {
+	db, mock, err := sqlmock.New()
+	s.Require().NoError(err)
+
+	mock.ExpectQuery(`SELECT "name" FROM "items"`).
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("hello"))
+
+	rows, err := db.Query(`SELECT "name" FROM "items"`)
+	s.Require().NoError(err)
+
+	sc := NewScanner(rows)
+	s.True(sc.Next())
+
+	var ns sql.NullString
+	err = sc.ScanVal(&ns)
+	s.Require().NoError(err)
+	s.Equal(sql.NullString{String: "hello", Valid: true}, ns)
+}