Browse Source

优化TypeUnion的描述方式

pull/15/head
Sydonian 2 years ago
parent
commit
e0118e5df8
9 changed files with 236 additions and 108 deletions
  1. +3
    -3
      api/scheduler/scheduler_test.go
  2. +20
    -11
      api/unifyops/unifyops.go
  3. +28
    -27
      models/job.go
  4. +15
    -14
      models/unifyops.go
  5. +41
    -20
      pkgs/mq/message.go
  6. +3
    -3
      pkgs/mq/message_test.go
  7. +31
    -0
      pkgs/types/union.go
  8. +45
    -19
      utils/serder/serder.go
  9. +50
    -11
      utils/serder/serder_test.go

+ 3
- 3
api/scheduler/scheduler_test.go View File

@@ -22,14 +22,14 @@ func Test_JobSet(t *testing.T) {
models.NormalJobInfo{
Type: models.JobTypeNormal,
Files: models.JobFilesInfo{
Dataset: models.PackageFileInfo{
Dataset: models.PackageJobFileInfo{
Type: models.FileInfoTypePackage,
},
Code: models.LocalFileInfo{
Code: models.LocalJobFileInfo{
Type: models.FileInfoTypeLocalFile,
LocalPath: "code",
},
Image: models.ImageFileInfo{
Image: models.ImageJobFileInfo{
Type: models.FileInfoTypeImage,
},
},


+ 20
- 11
api/unifyops/unifyops.go View File

@@ -227,13 +227,7 @@ func (c *Client) GetMemoryData(node Node) (*models.MemoryResourceData, error) {
return nil, fmt.Errorf("unknow response content type: %s", contType)
}

type ResourceData struct {
Name string `json:"name"`
Total models.DetailType[any] `json:"total"`
Available models.DetailType[any] `json:"available"`
}

func (c *Client) GetIndicatorData(node Node) (*[]ResourceData, error) {
func (c *Client) GetIndicatorData(node Node) (*[]models.ResourceData, error) {
url, err := url.JoinPath(c.baseURL, "/cmdb/resApi/getIndicatorData")
if err != nil {
return nil, err
@@ -248,16 +242,31 @@ func (c *Client) GetIndicatorData(node Node) (*[]ResourceData, error) {
contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, myhttp.ContentTypeJSON) {

var codeResp response[[]ResourceData]
var codeResp response[[]map[string]any]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
return nil, fmt.Errorf("parsing response: %w", err)
}

if codeResp.Code == CORRECT_CODE {
return &codeResp.Data, nil
if codeResp.Code != CORRECT_CODE {
return nil, codeResp.ToError()
}

return nil, codeResp.ToError()
mapToObjOpt := serder.MapToObjectOption{
UnionTypes: []serder.TaggedUnionType{
models.ResourceDataTaggedTypeUnion,
},
}
var ret []models.ResourceData
for _, mp := range codeResp.Data {
var data models.ResourceData
err := serder.MapToObject(mp, &data, mapToObjOpt)
if err != nil {
return nil, err
}
ret = append(ret, data)
}

return &ret, nil
}

return nil, fmt.Errorf("unknow response content type: %s", contType)


+ 28
- 27
models/job.go View File

@@ -1,6 +1,7 @@
package models

import (
"gitlink.org.cn/cloudream/common/pkgs/types"
myreflect "gitlink.org.cn/cloudream/common/utils/reflect"
"gitlink.org.cn/cloudream/common/utils/serder"
)
@@ -21,15 +22,15 @@ type JobSetInfo struct {

type JobInfo interface{}

var JobInfoTypeUnion = serder.NewTypeUnion[JobInfo]("type",
serder.NewStringTypeResolver().
Add(JobTypeNormal, myreflect.TypeOf[NormalJobInfo]()).
Add(JobTypeResource, myreflect.TypeOf[ResourceJobInfo]()),
var JobInfoTypeUnion = types.NewTypeUnion[JobInfo](
myreflect.TypeOf[NormalJobInfo](),
myreflect.TypeOf[ResourceJobInfo](),
)
var JobInfoTaggedTypeUnion = serder.NewTaggedTypeUnion(JobInfoTypeUnion, "Type", "type")

type NormalJobInfo struct {
LocalJobID string `json:"localJobID"`
Type string `json:"type"`
Type string `json:"type" union:"Normal"`
Files JobFilesInfo `json:"files"`
Runtime JobRuntimeInfo `json:"runtime"`
Resources JobResourcesInfo `json:"resources"`
@@ -37,43 +38,43 @@ type NormalJobInfo struct {

type ResourceJobInfo struct {
LocalJobID string `json:"localJobID"`
Type string `json:"type"`
Type string `json:"type" union:"Resource"`
TargetLocalJobID string `json:"targetLocalJobID"`
}

type JobFilesInfo struct {
Dataset FileInfo `json:"dataset"`
Code FileInfo `json:"code"`
Image FileInfo `json:"image"`
Dataset JobFileInfo `json:"dataset"`
Code JobFileInfo `json:"code"`
Image JobFileInfo `json:"image"`
}

type FileInfo interface{}
type JobFileInfo interface{}

var FileInfoTypeUnion = serder.NewTypeUnion[FileInfo]("type",
serder.NewStringTypeResolver().
Add(FileInfoTypePackage, myreflect.TypeOf[PackageFileInfo]()).
Add(FileInfoTypeLocalFile, myreflect.TypeOf[LocalFileInfo]()).
Add(FileInfoTypeResource, myreflect.TypeOf[ResourceFileInfo]()).
Add(FileInfoTypeImage, myreflect.TypeOf[ImageFileInfo]()),
var FileInfoTypeUnion = types.NewTypeUnion[JobFileInfo](
myreflect.TypeOf[PackageJobFileInfo](),
myreflect.TypeOf[LocalJobFileInfo](),
myreflect.TypeOf[ResourceJobFileInfo](),
myreflect.TypeOf[ImageJobFileInfo](),
)
var FileInfoTaggedTypeUnion = serder.NewTaggedTypeUnion(FileInfoTypeUnion, "Type", "type")

type PackageFileInfo struct {
Type string `json:"type"`
type PackageJobFileInfo struct {
Type string `json:"type" union:"Package"`
PackageID int64 `json:"packageID"`
}

type LocalFileInfo struct {
Type string `json:"type"`
type LocalJobFileInfo struct {
Type string `json:"type" union:"LocalFile"`
LocalPath string `json:"localPath"`
}

type ResourceFileInfo struct {
Type string `json:"type"`
type ResourceJobFileInfo struct {
Type string `json:"type" union:"Resource"`
ResourceLocalJobID string `json:"resourceLocalJobID"`
}

type ImageFileInfo struct {
Type string `json:"type"`
type ImageJobFileInfo struct {
Type string `json:"type" union:"Image"`
ImageID string `json:"imageID"`
}

@@ -104,9 +105,9 @@ func JobSetInfoFromJSON(data []byte) (*JobSetInfo, error) {

var ret JobSetInfo
err := serder.MapToObject(mp, &ret, serder.MapToObjectOption{
UnionTypes: []serder.UnionTypeInfo{
JobInfoTypeUnion,
FileInfoTypeUnion,
UnionTypes: []serder.TaggedUnionType{
JobInfoTaggedTypeUnion,
FileInfoTaggedTypeUnion,
},
})
if err != nil {


+ 15
- 14
models/unifyops.go View File

@@ -1,6 +1,7 @@
package models

import (
"gitlink.org.cn/cloudream/common/pkgs/types"
myreflect "gitlink.org.cn/cloudream/common/utils/reflect"
"gitlink.org.cn/cloudream/common/utils/serder"
)
@@ -25,15 +26,15 @@ type ResourceDataConst interface {
ResourceData | CPUResourceData | NPUResourceData | GPUResourceData | MLUResourceData | StorageResourceData | MemoryResourceData
}

var ResourceDataTypeUnion = serder.NewTypeUnion[ResourceData]("name",
serder.NewStringTypeResolver().
Add(ResourceTypeCPU, myreflect.TypeOf[CPUResourceData]()).
Add(ResourceTypeNPU, myreflect.TypeOf[NPUResourceData]()).
Add(ResourceTypeGPU, myreflect.TypeOf[GPUResourceData]()).
Add(ResourceTypeMLU, myreflect.TypeOf[MLUResourceData]()).
Add(ResourceTypeStorage, myreflect.TypeOf[StorageResourceData]()).
Add(ResourceTypeMemory, myreflect.TypeOf[MemoryResourceData]()),
var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData](
myreflect.TypeOf[CPUResourceData](),
myreflect.TypeOf[NPUResourceData](),
myreflect.TypeOf[GPUResourceData](),
myreflect.TypeOf[MLUResourceData](),
myreflect.TypeOf[StorageResourceData](),
myreflect.TypeOf[MemoryResourceData](),
)
var ResourceDataTaggedTypeUnion = serder.NewTaggedTypeUnion(ResourceDataTypeUnion, "Name", "name")

type DetailType[T any] struct {
Unit string `json:"unit"`
@@ -41,7 +42,7 @@ type DetailType[T any] struct {
}

type CPUResourceData struct {
Name string `json:"name"`
Name string `json:"name" union:"CPU"`
Total DetailType[int64] `json:"total"`
Available DetailType[int64] `json:"available"`
}
@@ -55,7 +56,7 @@ func NewCPUResourceData(name string, total DetailType[int64], available DetailTy
}

type NPUResourceData struct {
Name string `json:"name"`
Name string `json:"name" union:"NPU"`
Total DetailType[int64] `json:"total"`
Available DetailType[int64] `json:"available"`
}
@@ -69,7 +70,7 @@ func NewNPUResourceData(name string, total DetailType[int64], available DetailTy
}

type GPUResourceData struct {
Name string `json:"name"`
Name string `json:"name" union:"GPU"`
Total DetailType[int64] `json:"total"`
Available DetailType[int64] `json:"available"`
}
@@ -83,7 +84,7 @@ func NewGPUResourceData(name string, total DetailType[int64], available DetailTy
}

type MLUResourceData struct {
Name string `json:"name"`
Name string `json:"name" union:"MLU"`
Total DetailType[int64] `json:"total"`
Available DetailType[int64] `json:"available"`
}
@@ -97,7 +98,7 @@ func NewMLUResourceData(name string, total DetailType[int64], available DetailTy
}

type StorageResourceData struct {
Name string `json:"name"`
Name string `json:"name" union:"STORAGE"`
Total DetailType[float64] `json:"total"`
Available DetailType[float64] `json:"available"`
}
@@ -111,7 +112,7 @@ func NewStorageResourceData(name string, total DetailType[float64], available De
}

type MemoryResourceData struct {
Name string `json:"name"`
Name string `json:"name" union:"MEMORY"`
Total DetailType[float64] `json:"total"`
Available DetailType[float64] `json:"available"`
}


+ 41
- 20
pkgs/mq/message.go View File

@@ -7,8 +7,8 @@ import (
"unsafe"

jsoniter "github.com/json-iterator/go"
"gitlink.org.cn/cloudream/common/pkgs/types"
myreflect "gitlink.org.cn/cloudream/common/utils/reflect"
"gitlink.org.cn/cloudream/common/utils/serder"
)

const (
@@ -72,33 +72,49 @@ func MakeHeartbeatMessage() Message {
return msg
}

var unionTypes map[myreflect.Type]serder.UnionTypeInfo = make(map[reflect.Type]serder.UnionTypeInfo)
var messageBodyTypeUnionEles *serder.TypeNameResolver
type TypeUnionWithTypeName struct {
Union types.TypeUnion
TypeNameToType map[string]myreflect.Type
}

func (u *TypeUnionWithTypeName) Register(typ myreflect.Type) {
u.Union.ElementTypes = append(msgBodyTypeUnion.Union.ElementTypes, typ)
u.TypeNameToType[makeFullTypeName(typ)] = typ
}

var msgBodyTypeUnion *TypeUnionWithTypeName

// 所有新定义的Message都需要在init中调用此函数
func RegisterMessage[T any]() {
messageBodyTypeUnionEles.Register(myreflect.TypeOf[T]())
msgBodyTypeUnion.Register(myreflect.TypeOf[T]())
}

// 在序列化结构体中包含的UnionType类型字段时,会将字段值的实际类型保存在序列化后的结果中。
// 在反序列化时,会根据类型信息重建原本的字段值。
//
// 注:不是采用在序列化后的数据中增加TypeFieldName指名的字段数据,因此会无视UnionTypeInfo中的这个字段的设定
func RegisterUnionType(set serder.UnionTypeInfo) {
unionTypes[set.UnionType] = set
func RegisterUnionType(union types.TypeUnion) *TypeUnionWithTypeName {
myUnion := &TypeUnionWithTypeName{
Union: union,
TypeNameToType: make(map[string]reflect.Type),
}

jsoniter.RegisterTypeEncoderFunc(set.UnionType.String(),
for _, typ := range union.ElementTypes {
myUnion.TypeNameToType[makeFullTypeName(typ)] = typ
}

jsoniter.RegisterTypeEncoderFunc(union.UnionType.String(),
func(ptr unsafe.Pointer, stream *jsoniter.Stream) {
// 此处无法变成*UnionType,只能强转为*any
val := *(*any)(ptr)
if val != nil {
stream.WriteArrayStart()
typeStr, err := set.ElementTypes.TypeToString(myreflect.TypeOfValue(val))
if err != nil {
stream.Error = err

valType := myreflect.TypeOfValue(val)
if !myUnion.Union.Include(valType) {
stream.Error = fmt.Errorf("type %v is not in union %v", valType, union.UnionType)
return
}
stream.WriteString(typeStr)

stream.WriteString(makeFullTypeName(valType))
stream.WriteRaw(",")
stream.WriteVal(val)
stream.WriteArrayEnd()
@@ -110,7 +126,7 @@ func RegisterUnionType(set serder.UnionTypeInfo) {
return false
})

jsoniter.RegisterTypeDecoderFunc(set.UnionType.String(),
jsoniter.RegisterTypeDecoderFunc(union.UnionType.String(),
func(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
// 此处无法变成*UnionType,只能强转为*any
vp := (*any)(ptr)
@@ -125,9 +141,9 @@ func RegisterUnionType(set serder.UnionTypeInfo) {
typeStr := iter.ReadString()
iter.ReadArray()

typ, err := set.ElementTypes.StringToType(typeStr)
if err != nil {
iter.ReportError("get type from string", err.Error())
typ, ok := myUnion.TypeNameToType[typeStr]
if !ok {
iter.ReportError("decode UnionType", fmt.Sprintf("unknow type string %s under %v", typeStr, union.UnionType))
return
}

@@ -137,10 +153,16 @@ func RegisterUnionType(set serder.UnionTypeInfo) {

iter.ReadArray()
} else {
iter.ReportError("parse TypeSet field", fmt.Sprintf("unknow next token type %v", nextTkType))
iter.ReportError("decode UnionType", fmt.Sprintf("unknow next token type %v", nextTkType))
return
}
})

return myUnion
}

func makeFullTypeName(typ myreflect.Type) string {
return fmt.Sprintf("%s.%s", typ.PkgPath(), typ.Name())
}

/*
@@ -246,6 +268,5 @@ func Deserialize(data []byte) (*Message, error) {
}

func init() {
messageBodyTypeUnionEles = serder.NewTypeNameResolver(true)
RegisterUnionType(serder.NewTypeUnion[MessageBody]("", messageBodyTypeUnionEles))
msgBodyTypeUnion = RegisterUnionType(types.NewTypeUnion[MessageBody]())
}

+ 3
- 3
pkgs/mq/message_test.go View File

@@ -8,8 +8,8 @@ import (

jsoniter "github.com/json-iterator/go"
. "github.com/smartystreets/goconvey/convey"
"gitlink.org.cn/cloudream/common/pkgs/types"
myreflect "gitlink.org.cn/cloudream/common/utils/reflect"
"gitlink.org.cn/cloudream/common/utils/serder"
)

func TestMessage(t *testing.T) {
@@ -140,7 +140,7 @@ func TestMessage(t *testing.T) {
Value MyTypeSet
}
RegisterMessage[Body]()
RegisterUnionType(serder.NewTypeUnion[MyTypeSet]("", serder.NewTypeNameResolver(true)))
RegisterUnionType(types.NewTypeUnion[MyTypeSet]())

msg := MakeAppDataMessage(Body{Value: nil})
data, err := Serialize(msg)
@@ -159,7 +159,7 @@ func TestMessage(t *testing.T) {
Value MyTypeSet
}
RegisterMessage[Body]()
RegisterUnionType(serder.NewTypeUnion[MyTypeSet]("", serder.NewTypeNameResolver(true)))
RegisterUnionType(types.NewTypeUnion[MyTypeSet]())

msg := MakeAppDataMessage(Body{Value: struct{}{}})
_, err := Serialize(msg)


+ 31
- 0
pkgs/types/union.go View File

@@ -0,0 +1,31 @@
package types

import (
myreflect "gitlink.org.cn/cloudream/common/utils/reflect"
)

type TypeUnion struct {
UnionType myreflect.Type
ElementTypes []myreflect.Type
}

func NewTypeUnion[TU any](eleTypes ...myreflect.Type) TypeUnion {
return TypeUnion{
UnionType: myreflect.TypeOf[TU](),
ElementTypes: eleTypes,
}
}

func (u *TypeUnion) Include(typ myreflect.Type) bool {
for _, t := range u.ElementTypes {
if t == typ {
return true
}
}

return false
}

func (u *TypeUnion) Add(typ myreflect.Type) {
u.ElementTypes = append(u.ElementTypes, typ)
}

+ 45
- 19
utils/serder/serder.go View File

@@ -7,7 +7,7 @@ import (
"reflect"
"strings"

myreflect "gitlink.org.cn/cloudream/common/utils/reflect"
"gitlink.org.cn/cloudream/common/pkgs/types"
)

func ObjectToJSON(obj any) ([]byte, error) {
@@ -49,22 +49,48 @@ type TypeResolver interface {
StringToType(typeStr string) (reflect.Type, error)
}

type UnionTypeInfo struct {
UnionType reflect.Type
TypeFieldName string
ElementTypes TypeResolver
type TaggedUnionType struct {
Union types.TypeUnion
StrcutTagField string
JSONTagField string
TagToType map[string]reflect.Type
}

func NewTypeUnion[TU any](typeField string, eleTypes TypeResolver) UnionTypeInfo {
return UnionTypeInfo{
UnionType: myreflect.TypeOf[TU](),
TypeFieldName: typeField,
ElementTypes: eleTypes,
// 根据指定的字段的值来区分不同的类型。值可以通过在字段增加“union”Tag来指定。如果没有指定,则使用类型名。
func NewTaggedTypeUnion(union types.TypeUnion, structTagField string, jsonTagField string) TaggedUnionType {
tagToType := make(map[string]reflect.Type)

for _, typ := range union.ElementTypes {
if structTagField == "" {
tagToType[typ.Name()] = typ
continue
}

field, ok := typ.FieldByName(structTagField)
if !ok {
tagToType[typ.Name()] = typ
continue
}

tag := field.Tag.Get("union")
if tag == "" {
tagToType[typ.Name()] = typ
continue
}

tagToType[tag] = typ
}

return TaggedUnionType{
Union: union,
StrcutTagField: structTagField,
JSONTagField: jsonTagField,
TagToType: tagToType,
}
}

type MapToObjectOption struct {
UnionTypes []UnionTypeInfo // 转换过程中遇到这些类型时,会依据指定的字段的值,来决定转换后的实际类型
UnionTypes []TaggedUnionType // 转换过程中遇到这些类型时,会依据指定的字段的值,来决定转换后的实际类型
}

func MapToObject(m map[string]any, obj any, opt ...MapToObjectOption) error {
@@ -73,11 +99,11 @@ func MapToObject(m map[string]any, obj any, opt ...MapToObjectOption) error {
op = opt[0]
}

unionTypeMapping := make(map[reflect.Type]*UnionTypeInfo)
unionTypeMapping := make(map[reflect.Type]*TaggedUnionType)

for _, u := range op.UnionTypes {
uu := u
unionTypeMapping[u.UnionType] = &uu
unionTypeMapping[u.Union.UnionType] = &uu
}

convs := []Converter{
@@ -88,19 +114,19 @@ func MapToObject(m map[string]any, obj any, opt ...MapToObjectOption) error {
}

mp := from.Interface().(map[string]any)
tag, ok := mp[info.TypeFieldName]
tag, ok := mp[info.JSONTagField]
if !ok {
return nil, fmt.Errorf("converting to %v: no tag field %s in map", to.Type(), info.TypeFieldName)
return nil, fmt.Errorf("converting to %v: no tag field %s in map", to.Type(), info.JSONTagField)
}

tagStr, ok := tag.(string)
if !ok {
return nil, fmt.Errorf("converting to %v: tag field %s value is %v, which is not a string", to.Type(), info.TypeFieldName, tag)
return nil, fmt.Errorf("converting to %v: tag field %s value is %v, which is not a string", to.Type(), info.JSONTagField, tag)
}

eleType, err := info.ElementTypes.StringToType(tagStr)
if err != nil {
return nil, fmt.Errorf("converting to %v: %w", to.Type(), err)
eleType, ok := info.TagToType[tagStr]
if !ok {
return nil, fmt.Errorf("converting to %v: unknow type tag %s", to.Type(), tagStr)
}

to.Set(reflect.Indirect(reflect.New(eleType)))


+ 50
- 11
utils/serder/serder_test.go View File

@@ -6,6 +6,7 @@ import (
"testing"

. "github.com/smartystreets/goconvey/convey"
"gitlink.org.cn/cloudream/common/pkgs/types"
myreflect "gitlink.org.cn/cloudream/common/utils/reflect"
)

@@ -361,11 +362,13 @@ func Test_MapToObject(t *testing.T) {
type UnionType interface{}

type EleType1 struct {
Type string `json:"type" union:"1"`
Value1 string `json:"value1"`
}

type EleType2 struct {
Value2 int `json:"value2"`
Type string `json:"type" union:"2"`
Value2 int `json:"value2"`
}

type St struct {
@@ -387,22 +390,58 @@ func Test_MapToObject(t *testing.T) {

var ret St
err := MapToObject(mp, &ret, MapToObjectOption{
UnionTypes: []UnionTypeInfo{
{
UnionType: myreflect.TypeOf[UnionType](),
TypeFieldName: "type",
ElementTypes: NewStringTypeResolver().
Add("1", myreflect.TypeOf[EleType1]()).
Add("2", myreflect.TypeOf[EleType2]()),
},
UnionTypes: []TaggedUnionType{
NewTaggedTypeUnion(types.NewTypeUnion[UnionType](
myreflect.TypeOf[EleType1](),
myreflect.TypeOf[EleType2](),
),
"Type",
"type",
),
},
})

So(err, ShouldBeNil)

So(ret.Us, ShouldResemble, []UnionType{
EleType1{Value1: "1"},
EleType2{Value2: 2},
EleType1{Type: "1", Value1: "1"},
EleType2{Type: "2", Value2: 2},
})
})

Convey("要转换到的结构体就是一个UnionType", t, func() {
type UnionType interface{}

type EleType1 struct {
Type string `json:"type" union:"1"`
Value1 string `json:"value1"`
}

type EleType2 struct {
Type string `json:"type" union:"2"`
Value2 int `json:"value2"`
}

mp := map[string]any{
"type": "1",
"value1": "1",
}

var ret UnionType
err := MapToObject(mp, &ret, MapToObjectOption{
UnionTypes: []TaggedUnionType{
NewTaggedTypeUnion(types.NewTypeUnion[UnionType](
myreflect.TypeOf[EleType1](),
myreflect.TypeOf[EleType2](),
),
"Type",
"type",
),
},
})

So(err, ShouldBeNil)

So(ret, ShouldResemble, EleType1{Type: "1", Value1: "1"})
})
}

Loading…
Cancel
Save