diff --git a/api/scheduler/scheduler_test.go b/api/scheduler/scheduler_test.go index 1bcf2ff..b3ddb5d 100644 --- a/api/scheduler/scheduler_test.go +++ b/api/scheduler/scheduler_test.go @@ -22,14 +22,14 @@ func Test_JobSet(t *testing.T) { models.NormalJobInfo{ Type: models.JobTypeNormal, Files: models.JobFilesInfo{ - Dataset: models.PackageFileInfo{ + Dataset: models.PackageJobFileInfo{ Type: models.FileInfoTypePackage, }, - Code: models.LocalFileInfo{ + Code: models.LocalJobFileInfo{ Type: models.FileInfoTypeLocalFile, LocalPath: "code", }, - Image: models.ImageFileInfo{ + Image: models.ImageJobFileInfo{ Type: models.FileInfoTypeImage, }, }, diff --git a/api/unifyops/unifyops.go b/api/unifyops/unifyops.go index 7faaa92..c67fed0 100644 --- a/api/unifyops/unifyops.go +++ b/api/unifyops/unifyops.go @@ -227,13 +227,7 @@ func (c *Client) GetMemoryData(node Node) (*models.MemoryResourceData, error) { return nil, fmt.Errorf("unknow response content type: %s", contType) } -type ResourceData struct { - Name string `json:"name"` - Total models.DetailType[any] `json:"total"` - Available models.DetailType[any] `json:"available"` -} - -func (c *Client) GetIndicatorData(node Node) (*[]ResourceData, error) { +func (c *Client) GetIndicatorData(node Node) (*[]models.ResourceData, error) { url, err := url.JoinPath(c.baseURL, "/cmdb/resApi/getIndicatorData") if err != nil { return nil, err @@ -248,16 +242,31 @@ func (c *Client) GetIndicatorData(node Node) (*[]ResourceData, error) { contType := resp.Header.Get("Content-Type") if strings.Contains(contType, myhttp.ContentTypeJSON) { - var codeResp response[[]ResourceData] + var codeResp response[[]map[string]any] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { return nil, fmt.Errorf("parsing response: %w", err) } - if codeResp.Code == CORRECT_CODE { - return &codeResp.Data, nil + if codeResp.Code != CORRECT_CODE { + return nil, codeResp.ToError() } - return nil, codeResp.ToError() + mapToObjOpt := serder.MapToObjectOption{ + UnionTypes: []serder.TaggedUnionType{ + models.ResourceDataTaggedTypeUnion, + }, + } + var ret []models.ResourceData + for _, mp := range codeResp.Data { + var data models.ResourceData + err := serder.MapToObject(mp, &data, mapToObjOpt) + if err != nil { + return nil, err + } + ret = append(ret, data) + } + + return &ret, nil } return nil, fmt.Errorf("unknow response content type: %s", contType) diff --git a/models/job.go b/models/job.go index eccca17..94c0ca2 100644 --- a/models/job.go +++ b/models/job.go @@ -1,6 +1,7 @@ package models import ( + "gitlink.org.cn/cloudream/common/pkgs/types" myreflect "gitlink.org.cn/cloudream/common/utils/reflect" "gitlink.org.cn/cloudream/common/utils/serder" ) @@ -21,15 +22,15 @@ type JobSetInfo struct { type JobInfo interface{} -var JobInfoTypeUnion = serder.NewTypeUnion[JobInfo]("type", - serder.NewStringTypeResolver(). - Add(JobTypeNormal, myreflect.TypeOf[NormalJobInfo]()). - Add(JobTypeResource, myreflect.TypeOf[ResourceJobInfo]()), +var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( + myreflect.TypeOf[NormalJobInfo](), + myreflect.TypeOf[ResourceJobInfo](), ) +var JobInfoTaggedTypeUnion = serder.NewTaggedTypeUnion(JobInfoTypeUnion, "Type", "type") type NormalJobInfo struct { LocalJobID string `json:"localJobID"` - Type string `json:"type"` + Type string `json:"type" union:"Normal"` Files JobFilesInfo `json:"files"` Runtime JobRuntimeInfo `json:"runtime"` Resources JobResourcesInfo `json:"resources"` @@ -37,43 +38,43 @@ type NormalJobInfo struct { type ResourceJobInfo struct { LocalJobID string `json:"localJobID"` - Type string `json:"type"` + Type string `json:"type" union:"Resource"` TargetLocalJobID string `json:"targetLocalJobID"` } type JobFilesInfo struct { - Dataset FileInfo `json:"dataset"` - Code FileInfo `json:"code"` - Image FileInfo `json:"image"` + Dataset JobFileInfo `json:"dataset"` + Code JobFileInfo `json:"code"` + Image JobFileInfo `json:"image"` } -type FileInfo interface{} +type JobFileInfo interface{} -var FileInfoTypeUnion = serder.NewTypeUnion[FileInfo]("type", - serder.NewStringTypeResolver(). - Add(FileInfoTypePackage, myreflect.TypeOf[PackageFileInfo]()). - Add(FileInfoTypeLocalFile, myreflect.TypeOf[LocalFileInfo]()). - Add(FileInfoTypeResource, myreflect.TypeOf[ResourceFileInfo]()). - Add(FileInfoTypeImage, myreflect.TypeOf[ImageFileInfo]()), +var FileInfoTypeUnion = types.NewTypeUnion[JobFileInfo]( + myreflect.TypeOf[PackageJobFileInfo](), + myreflect.TypeOf[LocalJobFileInfo](), + myreflect.TypeOf[ResourceJobFileInfo](), + myreflect.TypeOf[ImageJobFileInfo](), ) +var FileInfoTaggedTypeUnion = serder.NewTaggedTypeUnion(FileInfoTypeUnion, "Type", "type") -type PackageFileInfo struct { - Type string `json:"type"` +type PackageJobFileInfo struct { + Type string `json:"type" union:"Package"` PackageID int64 `json:"packageID"` } -type LocalFileInfo struct { - Type string `json:"type"` +type LocalJobFileInfo struct { + Type string `json:"type" union:"LocalFile"` LocalPath string `json:"localPath"` } -type ResourceFileInfo struct { - Type string `json:"type"` +type ResourceJobFileInfo struct { + Type string `json:"type" union:"Resource"` ResourceLocalJobID string `json:"resourceLocalJobID"` } -type ImageFileInfo struct { - Type string `json:"type"` +type ImageJobFileInfo struct { + Type string `json:"type" union:"Image"` ImageID string `json:"imageID"` } @@ -104,9 +105,9 @@ func JobSetInfoFromJSON(data []byte) (*JobSetInfo, error) { var ret JobSetInfo err := serder.MapToObject(mp, &ret, serder.MapToObjectOption{ - UnionTypes: []serder.UnionTypeInfo{ - JobInfoTypeUnion, - FileInfoTypeUnion, + UnionTypes: []serder.TaggedUnionType{ + JobInfoTaggedTypeUnion, + FileInfoTaggedTypeUnion, }, }) if err != nil { diff --git a/models/unifyops.go b/models/unifyops.go index bf717f3..f5a83ab 100644 --- a/models/unifyops.go +++ b/models/unifyops.go @@ -1,6 +1,7 @@ package models import ( + "gitlink.org.cn/cloudream/common/pkgs/types" myreflect "gitlink.org.cn/cloudream/common/utils/reflect" "gitlink.org.cn/cloudream/common/utils/serder" ) @@ -25,15 +26,15 @@ type ResourceDataConst interface { ResourceData | CPUResourceData | NPUResourceData | GPUResourceData | MLUResourceData | StorageResourceData | MemoryResourceData } -var ResourceDataTypeUnion = serder.NewTypeUnion[ResourceData]("name", - serder.NewStringTypeResolver(). - Add(ResourceTypeCPU, myreflect.TypeOf[CPUResourceData]()). - Add(ResourceTypeNPU, myreflect.TypeOf[NPUResourceData]()). - Add(ResourceTypeGPU, myreflect.TypeOf[GPUResourceData]()). - Add(ResourceTypeMLU, myreflect.TypeOf[MLUResourceData]()). - Add(ResourceTypeStorage, myreflect.TypeOf[StorageResourceData]()). - Add(ResourceTypeMemory, myreflect.TypeOf[MemoryResourceData]()), +var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( + myreflect.TypeOf[CPUResourceData](), + myreflect.TypeOf[NPUResourceData](), + myreflect.TypeOf[GPUResourceData](), + myreflect.TypeOf[MLUResourceData](), + myreflect.TypeOf[StorageResourceData](), + myreflect.TypeOf[MemoryResourceData](), ) +var ResourceDataTaggedTypeUnion = serder.NewTaggedTypeUnion(ResourceDataTypeUnion, "Name", "name") type DetailType[T any] struct { Unit string `json:"unit"` @@ -41,7 +42,7 @@ type DetailType[T any] struct { } type CPUResourceData struct { - Name string `json:"name"` + Name string `json:"name" union:"CPU"` Total DetailType[int64] `json:"total"` Available DetailType[int64] `json:"available"` } @@ -55,7 +56,7 @@ func NewCPUResourceData(name string, total DetailType[int64], available DetailTy } type NPUResourceData struct { - Name string `json:"name"` + Name string `json:"name" union:"NPU"` Total DetailType[int64] `json:"total"` Available DetailType[int64] `json:"available"` } @@ -69,7 +70,7 @@ func NewNPUResourceData(name string, total DetailType[int64], available DetailTy } type GPUResourceData struct { - Name string `json:"name"` + Name string `json:"name" union:"GPU"` Total DetailType[int64] `json:"total"` Available DetailType[int64] `json:"available"` } @@ -83,7 +84,7 @@ func NewGPUResourceData(name string, total DetailType[int64], available DetailTy } type MLUResourceData struct { - Name string `json:"name"` + Name string `json:"name" union:"MLU"` Total DetailType[int64] `json:"total"` Available DetailType[int64] `json:"available"` } @@ -97,7 +98,7 @@ func NewMLUResourceData(name string, total DetailType[int64], available DetailTy } type StorageResourceData struct { - Name string `json:"name"` + Name string `json:"name" union:"STORAGE"` Total DetailType[float64] `json:"total"` Available DetailType[float64] `json:"available"` } @@ -111,7 +112,7 @@ func NewStorageResourceData(name string, total DetailType[float64], available De } type MemoryResourceData struct { - Name string `json:"name"` + Name string `json:"name" union:"MEMORY"` Total DetailType[float64] `json:"total"` Available DetailType[float64] `json:"available"` } diff --git a/pkgs/mq/message.go b/pkgs/mq/message.go index 2af9be3..35d9a16 100644 --- a/pkgs/mq/message.go +++ b/pkgs/mq/message.go @@ -7,8 +7,8 @@ import ( "unsafe" jsoniter "github.com/json-iterator/go" + "gitlink.org.cn/cloudream/common/pkgs/types" myreflect "gitlink.org.cn/cloudream/common/utils/reflect" - "gitlink.org.cn/cloudream/common/utils/serder" ) const ( @@ -72,33 +72,49 @@ func MakeHeartbeatMessage() Message { return msg } -var unionTypes map[myreflect.Type]serder.UnionTypeInfo = make(map[reflect.Type]serder.UnionTypeInfo) -var messageBodyTypeUnionEles *serder.TypeNameResolver +type TypeUnionWithTypeName struct { + Union types.TypeUnion + TypeNameToType map[string]myreflect.Type +} + +func (u *TypeUnionWithTypeName) Register(typ myreflect.Type) { + u.Union.ElementTypes = append(msgBodyTypeUnion.Union.ElementTypes, typ) + u.TypeNameToType[makeFullTypeName(typ)] = typ +} + +var msgBodyTypeUnion *TypeUnionWithTypeName // 所有新定义的Message都需要在init中调用此函数 func RegisterMessage[T any]() { - messageBodyTypeUnionEles.Register(myreflect.TypeOf[T]()) + msgBodyTypeUnion.Register(myreflect.TypeOf[T]()) } // 在序列化结构体中包含的UnionType类型字段时,会将字段值的实际类型保存在序列化后的结果中。 // 在反序列化时,会根据类型信息重建原本的字段值。 -// -// 注:不是采用在序列化后的数据中增加TypeFieldName指名的字段数据,因此会无视UnionTypeInfo中的这个字段的设定 -func RegisterUnionType(set serder.UnionTypeInfo) { - unionTypes[set.UnionType] = set +func RegisterUnionType(union types.TypeUnion) *TypeUnionWithTypeName { + myUnion := &TypeUnionWithTypeName{ + Union: union, + TypeNameToType: make(map[string]reflect.Type), + } - jsoniter.RegisterTypeEncoderFunc(set.UnionType.String(), + for _, typ := range union.ElementTypes { + myUnion.TypeNameToType[makeFullTypeName(typ)] = typ + } + + jsoniter.RegisterTypeEncoderFunc(union.UnionType.String(), func(ptr unsafe.Pointer, stream *jsoniter.Stream) { // 此处无法变成*UnionType,只能强转为*any val := *(*any)(ptr) if val != nil { stream.WriteArrayStart() - typeStr, err := set.ElementTypes.TypeToString(myreflect.TypeOfValue(val)) - if err != nil { - stream.Error = err + + valType := myreflect.TypeOfValue(val) + if !myUnion.Union.Include(valType) { + stream.Error = fmt.Errorf("type %v is not in union %v", valType, union.UnionType) return } - stream.WriteString(typeStr) + + stream.WriteString(makeFullTypeName(valType)) stream.WriteRaw(",") stream.WriteVal(val) stream.WriteArrayEnd() @@ -110,7 +126,7 @@ func RegisterUnionType(set serder.UnionTypeInfo) { return false }) - jsoniter.RegisterTypeDecoderFunc(set.UnionType.String(), + jsoniter.RegisterTypeDecoderFunc(union.UnionType.String(), func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { // 此处无法变成*UnionType,只能强转为*any vp := (*any)(ptr) @@ -125,9 +141,9 @@ func RegisterUnionType(set serder.UnionTypeInfo) { typeStr := iter.ReadString() iter.ReadArray() - typ, err := set.ElementTypes.StringToType(typeStr) - if err != nil { - iter.ReportError("get type from string", err.Error()) + typ, ok := myUnion.TypeNameToType[typeStr] + if !ok { + iter.ReportError("decode UnionType", fmt.Sprintf("unknow type string %s under %v", typeStr, union.UnionType)) return } @@ -137,10 +153,16 @@ func RegisterUnionType(set serder.UnionTypeInfo) { iter.ReadArray() } else { - iter.ReportError("parse TypeSet field", fmt.Sprintf("unknow next token type %v", nextTkType)) + iter.ReportError("decode UnionType", fmt.Sprintf("unknow next token type %v", nextTkType)) return } }) + + return myUnion +} + +func makeFullTypeName(typ myreflect.Type) string { + return fmt.Sprintf("%s.%s", typ.PkgPath(), typ.Name()) } /* @@ -246,6 +268,5 @@ func Deserialize(data []byte) (*Message, error) { } func init() { - messageBodyTypeUnionEles = serder.NewTypeNameResolver(true) - RegisterUnionType(serder.NewTypeUnion[MessageBody]("", messageBodyTypeUnionEles)) + msgBodyTypeUnion = RegisterUnionType(types.NewTypeUnion[MessageBody]()) } diff --git a/pkgs/mq/message_test.go b/pkgs/mq/message_test.go index dffd508..f04ee77 100644 --- a/pkgs/mq/message_test.go +++ b/pkgs/mq/message_test.go @@ -8,8 +8,8 @@ import ( jsoniter "github.com/json-iterator/go" . "github.com/smartystreets/goconvey/convey" + "gitlink.org.cn/cloudream/common/pkgs/types" myreflect "gitlink.org.cn/cloudream/common/utils/reflect" - "gitlink.org.cn/cloudream/common/utils/serder" ) func TestMessage(t *testing.T) { @@ -140,7 +140,7 @@ func TestMessage(t *testing.T) { Value MyTypeSet } RegisterMessage[Body]() - RegisterUnionType(serder.NewTypeUnion[MyTypeSet]("", serder.NewTypeNameResolver(true))) + RegisterUnionType(types.NewTypeUnion[MyTypeSet]()) msg := MakeAppDataMessage(Body{Value: nil}) data, err := Serialize(msg) @@ -159,7 +159,7 @@ func TestMessage(t *testing.T) { Value MyTypeSet } RegisterMessage[Body]() - RegisterUnionType(serder.NewTypeUnion[MyTypeSet]("", serder.NewTypeNameResolver(true))) + RegisterUnionType(types.NewTypeUnion[MyTypeSet]()) msg := MakeAppDataMessage(Body{Value: struct{}{}}) _, err := Serialize(msg) diff --git a/pkgs/types/union.go b/pkgs/types/union.go new file mode 100644 index 0000000..e073d9c --- /dev/null +++ b/pkgs/types/union.go @@ -0,0 +1,31 @@ +package types + +import ( + myreflect "gitlink.org.cn/cloudream/common/utils/reflect" +) + +type TypeUnion struct { + UnionType myreflect.Type + ElementTypes []myreflect.Type +} + +func NewTypeUnion[TU any](eleTypes ...myreflect.Type) TypeUnion { + return TypeUnion{ + UnionType: myreflect.TypeOf[TU](), + ElementTypes: eleTypes, + } +} + +func (u *TypeUnion) Include(typ myreflect.Type) bool { + for _, t := range u.ElementTypes { + if t == typ { + return true + } + } + + return false +} + +func (u *TypeUnion) Add(typ myreflect.Type) { + u.ElementTypes = append(u.ElementTypes, typ) +} diff --git a/utils/serder/serder.go b/utils/serder/serder.go index e63fb5a..26e932d 100644 --- a/utils/serder/serder.go +++ b/utils/serder/serder.go @@ -7,7 +7,7 @@ import ( "reflect" "strings" - myreflect "gitlink.org.cn/cloudream/common/utils/reflect" + "gitlink.org.cn/cloudream/common/pkgs/types" ) func ObjectToJSON(obj any) ([]byte, error) { @@ -49,22 +49,48 @@ type TypeResolver interface { StringToType(typeStr string) (reflect.Type, error) } -type UnionTypeInfo struct { - UnionType reflect.Type - TypeFieldName string - ElementTypes TypeResolver +type TaggedUnionType struct { + Union types.TypeUnion + StrcutTagField string + JSONTagField string + TagToType map[string]reflect.Type } -func NewTypeUnion[TU any](typeField string, eleTypes TypeResolver) UnionTypeInfo { - return UnionTypeInfo{ - UnionType: myreflect.TypeOf[TU](), - TypeFieldName: typeField, - ElementTypes: eleTypes, +// 根据指定的字段的值来区分不同的类型。值可以通过在字段增加“union”Tag来指定。如果没有指定,则使用类型名。 +func NewTaggedTypeUnion(union types.TypeUnion, structTagField string, jsonTagField string) TaggedUnionType { + tagToType := make(map[string]reflect.Type) + + for _, typ := range union.ElementTypes { + if structTagField == "" { + tagToType[typ.Name()] = typ + continue + } + + field, ok := typ.FieldByName(structTagField) + if !ok { + tagToType[typ.Name()] = typ + continue + } + + tag := field.Tag.Get("union") + if tag == "" { + tagToType[typ.Name()] = typ + continue + } + + tagToType[tag] = typ + } + + return TaggedUnionType{ + Union: union, + StrcutTagField: structTagField, + JSONTagField: jsonTagField, + TagToType: tagToType, } } type MapToObjectOption struct { - UnionTypes []UnionTypeInfo // 转换过程中遇到这些类型时,会依据指定的字段的值,来决定转换后的实际类型 + UnionTypes []TaggedUnionType // 转换过程中遇到这些类型时,会依据指定的字段的值,来决定转换后的实际类型 } func MapToObject(m map[string]any, obj any, opt ...MapToObjectOption) error { @@ -73,11 +99,11 @@ func MapToObject(m map[string]any, obj any, opt ...MapToObjectOption) error { op = opt[0] } - unionTypeMapping := make(map[reflect.Type]*UnionTypeInfo) + unionTypeMapping := make(map[reflect.Type]*TaggedUnionType) for _, u := range op.UnionTypes { uu := u - unionTypeMapping[u.UnionType] = &uu + unionTypeMapping[u.Union.UnionType] = &uu } convs := []Converter{ @@ -88,19 +114,19 @@ func MapToObject(m map[string]any, obj any, opt ...MapToObjectOption) error { } mp := from.Interface().(map[string]any) - tag, ok := mp[info.TypeFieldName] + tag, ok := mp[info.JSONTagField] if !ok { - return nil, fmt.Errorf("converting to %v: no tag field %s in map", to.Type(), info.TypeFieldName) + return nil, fmt.Errorf("converting to %v: no tag field %s in map", to.Type(), info.JSONTagField) } tagStr, ok := tag.(string) if !ok { - return nil, fmt.Errorf("converting to %v: tag field %s value is %v, which is not a string", to.Type(), info.TypeFieldName, tag) + return nil, fmt.Errorf("converting to %v: tag field %s value is %v, which is not a string", to.Type(), info.JSONTagField, tag) } - eleType, err := info.ElementTypes.StringToType(tagStr) - if err != nil { - return nil, fmt.Errorf("converting to %v: %w", to.Type(), err) + eleType, ok := info.TagToType[tagStr] + if !ok { + return nil, fmt.Errorf("converting to %v: unknow type tag %s", to.Type(), tagStr) } to.Set(reflect.Indirect(reflect.New(eleType))) diff --git a/utils/serder/serder_test.go b/utils/serder/serder_test.go index 72747fb..bd978be 100644 --- a/utils/serder/serder_test.go +++ b/utils/serder/serder_test.go @@ -6,6 +6,7 @@ import ( "testing" . "github.com/smartystreets/goconvey/convey" + "gitlink.org.cn/cloudream/common/pkgs/types" myreflect "gitlink.org.cn/cloudream/common/utils/reflect" ) @@ -361,11 +362,13 @@ func Test_MapToObject(t *testing.T) { type UnionType interface{} type EleType1 struct { + Type string `json:"type" union:"1"` Value1 string `json:"value1"` } type EleType2 struct { - Value2 int `json:"value2"` + Type string `json:"type" union:"2"` + Value2 int `json:"value2"` } type St struct { @@ -387,22 +390,58 @@ func Test_MapToObject(t *testing.T) { var ret St err := MapToObject(mp, &ret, MapToObjectOption{ - UnionTypes: []UnionTypeInfo{ - { - UnionType: myreflect.TypeOf[UnionType](), - TypeFieldName: "type", - ElementTypes: NewStringTypeResolver(). - Add("1", myreflect.TypeOf[EleType1]()). - Add("2", myreflect.TypeOf[EleType2]()), - }, + UnionTypes: []TaggedUnionType{ + NewTaggedTypeUnion(types.NewTypeUnion[UnionType]( + myreflect.TypeOf[EleType1](), + myreflect.TypeOf[EleType2](), + ), + "Type", + "type", + ), }, }) So(err, ShouldBeNil) So(ret.Us, ShouldResemble, []UnionType{ - EleType1{Value1: "1"}, - EleType2{Value2: 2}, + EleType1{Type: "1", Value1: "1"}, + EleType2{Type: "2", Value2: 2}, + }) + }) + + Convey("要转换到的结构体就是一个UnionType", t, func() { + type UnionType interface{} + + type EleType1 struct { + Type string `json:"type" union:"1"` + Value1 string `json:"value1"` + } + + type EleType2 struct { + Type string `json:"type" union:"2"` + Value2 int `json:"value2"` + } + + mp := map[string]any{ + "type": "1", + "value1": "1", + } + + var ret UnionType + err := MapToObject(mp, &ret, MapToObjectOption{ + UnionTypes: []TaggedUnionType{ + NewTaggedTypeUnion(types.NewTypeUnion[UnionType]( + myreflect.TypeOf[EleType1](), + myreflect.TypeOf[EleType2](), + ), + "Type", + "type", + ), + }, }) + + So(err, ShouldBeNil) + + So(ret, ShouldResemble, EleType1{Type: "1", Value1: "1"}) }) }