From e0c648bb8e782cf474c4ab823499cca591918aed Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 4 Dec 2024 16:33:28 +0800 Subject: [PATCH 001/155] =?UTF-8?q?=E6=96=B0=E5=A2=9EPCM=E9=A2=84=E8=B0=83?= =?UTF-8?q?=E5=BA=A6=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/client.go | 57 +++++++++ sdks/pcmscheduler/config.go | 5 + sdks/pcmscheduler/jobset.go | 144 ++++++++++++++++++++++ sdks/pcmscheduler/models.go | 238 ++++++++++++++++++++++++++++++++++++ 4 files changed, 444 insertions(+) create mode 100644 sdks/pcmscheduler/client.go create mode 100644 sdks/pcmscheduler/config.go create mode 100644 sdks/pcmscheduler/jobset.go create mode 100644 sdks/pcmscheduler/models.go diff --git a/sdks/pcmscheduler/client.go b/sdks/pcmscheduler/client.go new file mode 100644 index 0000000..a058dc7 --- /dev/null +++ b/sdks/pcmscheduler/client.go @@ -0,0 +1,57 @@ +package sch + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/pcmscheduler/config.go b/sdks/pcmscheduler/config.go new file mode 100644 index 0000000..07b0093 --- /dev/null +++ b/sdks/pcmscheduler/config.go @@ -0,0 +1,5 @@ +package sch + +type Config struct { + URL string `json:"url"` +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go new file mode 100644 index 0000000..ea4d69c --- /dev/null +++ b/sdks/pcmscheduler/jobset.go @@ -0,0 +1,144 @@ +package sch + +import ( + "fmt" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" +) + +type GetClusterInfoReq struct { + IDs []ClusterID `json:"ids"` +} + +func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/queryResources") + if err != nil { + return nil, err + } + resp, err := http2.GetJSON(targetUrl, http2.RequestParam{Body: req}) + if err != nil { + return nil, err + } + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + + var codeResp response[[]ClusterDetail] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return codeResp.Data, nil + } + + return nil, codeResp.ToError() + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} + +type CreateJobReq struct { + DataDistribute DataDistribute `json:"dataDistribute"` + Resources schsdk.JobResourcesInfo `json:"resources"` +} + +type DataDistribute struct { + Dataset []DatasetDistribute `json:"dataset"` + Code []CodeDistribute `json:"code"` + Image []ImageDistribute `json:"image"` + Model []ModelDistribute `json:"model"` +} + +type DatasetDistribute struct { + DataName string `json:"dataName"` + PackageID cdssdk.PackageID `json:"packageID"` + Clusters []ClusterID `json:"clusters"` +} + +type CodeDistribute struct { + DataName string `json:"dataName"` + PackageID cdssdk.PackageID `json:"packageID"` + Clusters []ClusterID `json:"clusters"` +} + +type ImageDistribute struct { + DataName string `json:"dataName"` + PackageID cdssdk.PackageID `json:"packageID"` + Clusters []ClusterID `json:"clusters"` +} + +type ModelDistribute struct { + DataName string `json:"dataName"` + PackageID cdssdk.PackageID `json:"packageID"` + Clusters []ClusterID `json:"clusters"` +} + +type Cluster struct { + ClusterID ClusterID `json:"clusterID"` + StorageID cdssdk.StorageID `json:"storageID"` +} + +type CreateJobResp struct { + TaskID TaskID `json:"taskID"` + ScheduleDatas []ScheduleData `json:"scheduleDatas"` +} + +type ScheduleData struct { + DataType string `json:"dataType"` + PackageID cdssdk.PackageID `json:"packageID"` + StorageType string `json:"storageType"` + ClusterIDs []ClusterID `json:"clusterIDs"` +} + +func (c *Client) CreateJob(req CreateJobReq) (CreateJobResp, error) { + +} + +type DataScheduleReq struct { + PackageID cdssdk.PackageID `json:"packageID"` + StorageType string `json:"storageType"` + Clusters []Cluster `json:"clusters"` +} + +type DataScheduleResp struct { + Results []DataScheduleResult `json:"results"` +} + +type DataScheduleResult struct { + ClusterID ClusterID `json:"clusterID"` + PackageID cdssdk.PackageID `json:"packageID"` + PackageFullPath string `json:"packageFullPath"` + Status bool `json:"status"` + Msg string `json:"msg"` +} + +func (c *Client) DataSchedule(req DataScheduleReq) (DataScheduleResp, error) { + +} + +type RunJobReq struct { + TaskID TaskID `json:"taskID"` + ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` +} + +type DataScheduleResults struct { + DataType string `json:"dataType"` + Results []DataScheduleResult `json:"results"` +} + +func (c *Client) RunJob(req RunJobReq) error { + +} + +type CancelJobReq struct { + TaskID TaskID `json:"taskID"` + Msg string `json:"msg"` +} + +func (c *Client) CancelJob(req CancelJobReq) error { + +} diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go new file mode 100644 index 0000000..e9bda19 --- /dev/null +++ b/sdks/pcmscheduler/models.go @@ -0,0 +1,238 @@ +package sch + +import ( + "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/utils/serder" +) + +type ResourceType string + +const ( + ResourceTypeCPU ResourceType = "CPU" + ResourceTypeNPU ResourceType = "NPU" + ResourceTypeGPU ResourceType = "GPU" + ResourceTypeMLU ResourceType = "MLU" + ResourceTypeStorage ResourceType = "STORAGE" + ResourceTypeMemory ResourceType = "MEMORY" + + CODE = "code" + DATASET = "dataset" + IMAGE = "image" + MODEL = "model" +) + +type ClusterID int64 +type TaskID int64 + +type ClusterDetail struct { + ID ClusterID `json:"id"` + Resources []ResourceData `json:"resources"` +} + +type ResourceData interface { + Noop() +} + +var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( + (*CPUResourceData)(nil), + (*NPUResourceData)(nil), + (*GPUResourceData)(nil), + (*MLUResourceData)(nil), + (*StorageResourceData)(nil), + (*MemoryResourceData)(nil), +) +var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "name") + +type ResourceDataBase struct{} + +func (d *ResourceDataBase) Noop() {} + +type UnitValue[T any] struct { + Unit string `json:"unit"` + Value T `json:"value"` +} + +type CPUResourceData struct { + serder.Metadata `union:"CPU"` + ResourceDataBase + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +func NewCPUResourceData(total UnitValue[int64], available UnitValue[int64]) *CPUResourceData { + return &CPUResourceData{ + Name: ResourceTypeCPU, + Total: total, + Available: available, + } +} + +type NPUResourceData struct { + serder.Metadata `union:"NPU"` + ResourceDataBase + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +func NewNPUResourceData(total UnitValue[int64], available UnitValue[int64]) *NPUResourceData { + return &NPUResourceData{ + Name: ResourceTypeNPU, + Total: total, + Available: available, + } +} + +type GPUResourceData struct { + serder.Metadata `union:"GPU"` + ResourceDataBase + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +func NewGPUResourceData(total UnitValue[int64], available UnitValue[int64]) *GPUResourceData { + return &GPUResourceData{ + Name: ResourceTypeGPU, + Total: total, + Available: available, + } +} + +type MLUResourceData struct { + serder.Metadata `union:"MLU"` + ResourceDataBase + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +func NewMLUResourceData(total UnitValue[int64], available UnitValue[int64]) *MLUResourceData { + return &MLUResourceData{ + Name: ResourceTypeMLU, + Total: total, + Available: available, + } +} + +type StorageResourceData struct { + serder.Metadata `union:"STORAGE"` + ResourceDataBase + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +func NewStorageResourceData(total UnitValue[float64], available UnitValue[float64]) *StorageResourceData { + return &StorageResourceData{ + Name: ResourceTypeStorage, + Total: total, + Available: available, + } +} + +type MemoryResourceData struct { + serder.Metadata `union:"MEMORY"` + ResourceDataBase + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +func NewMemoryResourceData(total UnitValue[float64], available UnitValue[float64]) *MemoryResourceData { + return &MemoryResourceData{ + Name: ResourceTypeMemory, + Total: total, + Available: available, + } +} + +type ResourcePriority interface { + Noop() +} + +type ResourcePriorityBase struct { +} + +var ResourcePriorityTypeUnion = types.NewTypeUnion[ResourcePriority]( + (*RegionPriority)(nil), + (*ChipPriority)(nil), + (*BiasPriority)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&ResourcePriorityTypeUnion, "type") + +func (d *ResourcePriorityBase) Noop() {} + +type RegionPriority struct { + serder.Metadata `union:"region"` + ResourcePriorityBase + Options []string `json:"options"` +} + +type ChipPriority struct { + serder.Metadata `union:"chip"` + ResourcePriorityBase + Options []string `json:"options"` +} + +type BiasPriority struct { + serder.Metadata `union:"bias"` + ResourcePriorityBase + Options []string `json:"options"` +} + +type UploadInfo interface { + Noop() +} + +var UploadInfoTypeUnion = types.NewTypeUnion[UploadInfo]( + (*LocalUploadInfo)(nil), + (*RemoteUploadInfo)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type") + +type LocalUploadInfo struct { + serder.Metadata `union:"local"` + UploadInfoBase + LocalPath string `json:"localPath"` +} + +type RemoteUploadInfo struct { + serder.Metadata `union:"url"` + UploadInfoBase + Url string `json:"url"` +} + +type UploadInfoBase struct{} + +func (d *UploadInfoBase) Noop() {} + +type UploadPriority interface { + Noop() +} + +var UploadPriorityTypeUnion = types.NewTypeUnion[UploadPriority]( + (*Preferences)(nil), + (*SpecifyCluster)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadPriorityTypeUnion, "type") + +type Preferences struct { + serder.Metadata `union:"preference"` + UploadPriorityBase + ResourcePriorities []ResourcePriority `json:"priorities"` +} + +type SpecifyCluster struct { + serder.Metadata `union:"specify"` + UploadPriorityBase + Clusters []ClusterID `json:"clusters"` +} + +type UploadPriorityBase struct{} + +func (d *UploadPriorityBase) Noop() {} From 171f54a6b2f916247af7370f08f9e0182e41b992 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 18 Dec 2024 15:28:28 +0800 Subject: [PATCH 002/155] =?UTF-8?q?=E6=8E=A5=E5=85=A5=E5=AD=98=E8=AF=81?= =?UTF-8?q?=E5=92=8Cuploader=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 34 +++++++ sdks/blockchain/client.go | 57 +++++++++++ sdks/blockchain/config.go | 9 ++ sdks/blockchain/models.go | 3 + sdks/blockchain/test.go | 37 +++++++ sdks/blockchain/uploader.go | 179 +++++++++++++++++++++++++++++++++ sdks/pcmscheduler/jobset.go | 139 +++++++++++++++++++------- sdks/pcmscheduler/models.go | 122 +++++++++++++---------- sdks/scheduler/models.go | 105 ++++++++++++++++++++ sdks/uploader/client.go | 57 +++++++++++ sdks/uploader/config.go | 5 + sdks/uploader/models.go | 28 ++++++ sdks/uploader/uploader.go | 182 ++++++++++++++++++++++++++++++++++ utils/config/config.go | 6 +- 14 files changed, 874 insertions(+), 89 deletions(-) create mode 100644 sdks/blockchain/blockchain.go create mode 100644 sdks/blockchain/client.go create mode 100644 sdks/blockchain/config.go create mode 100644 sdks/blockchain/models.go create mode 100644 sdks/blockchain/test.go create mode 100644 sdks/blockchain/uploader.go create mode 100644 sdks/uploader/client.go create mode 100644 sdks/uploader/config.go create mode 100644 sdks/uploader/models.go create mode 100644 sdks/uploader/uploader.go diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go new file mode 100644 index 0000000..9a51461 --- /dev/null +++ b/sdks/blockchain/blockchain.go @@ -0,0 +1,34 @@ +package blockchain + +import ( + "gitlink.org.cn/cloudream/common/utils/http2" + "net/url" +) + +type InvokeReq struct { + ContractAddress string `json:"contractAddress"` + FunctionName string `json:"functionName"` + MemberName string `json:"memberName"` + Type string `json:"type"` + Args []string `json:"args"` +} + +func (c *Client) BlockChainInvoke(req InvokeReq) error { + targetUrl, err := url.JoinPath(c.baseURL, "/contract/invoke") + if err != nil { + return err + } + + header := make(map[string]string) + header["Content-Type"] = http2.ContentTypeJSON + + _, err = http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: header, + }) + if err != nil { + return err + } + + return nil +} diff --git a/sdks/blockchain/client.go b/sdks/blockchain/client.go new file mode 100644 index 0000000..7121aa0 --- /dev/null +++ b/sdks/blockchain/client.go @@ -0,0 +1,57 @@ +package blockchain + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/blockchain/config.go b/sdks/blockchain/config.go new file mode 100644 index 0000000..59254f5 --- /dev/null +++ b/sdks/blockchain/config.go @@ -0,0 +1,9 @@ +package blockchain + +type Config struct { + URL string `json:"url"` + ContractAddress string `json:"contractAddress"` + FunctionName string `json:"functionName"` + MemberName string `json:"memberName"` + Type string `json:"type"` +} diff --git a/sdks/blockchain/models.go b/sdks/blockchain/models.go new file mode 100644 index 0000000..8da50d0 --- /dev/null +++ b/sdks/blockchain/models.go @@ -0,0 +1,3 @@ +package blockchain + +type ClusterID string diff --git a/sdks/blockchain/test.go b/sdks/blockchain/test.go new file mode 100644 index 0000000..8a85711 --- /dev/null +++ b/sdks/blockchain/test.go @@ -0,0 +1,37 @@ +package blockchain + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" +) + +func main() { + url := "http://localhost:2006/contract/invoke" + method := "POST" + payload := strings.NewReader(`{` + " " + ` "contractAddress" : "0xc860ab27901b3c2b810165a6096c64d88763617f",` + " " + ` "functionName" : "storeEvidence",` + " " + ` "args" : ["3","touteng"],` + " " + ` "memberName" :"pcm",` + " " + ` "type": "2"` + " " + ` }`) + client := &http.Client{} + req, err := http.NewRequest(method, url, payload) + if err != nil { + fmt.Println(err) + return + } + req.Header.Add("User-Agent", "Apifox/1.0.0 (https://apifox.com)") + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "*/*") + req.Header.Add("Host", "localhost:2006") + req.Header.Add("Connection", "keep-alive") + res, err := client.Do(req) + if err != nil { + fmt.Println(err) + return + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + return + } + fmt.Println(string(body)) +} diff --git a/sdks/blockchain/uploader.go b/sdks/blockchain/uploader.go new file mode 100644 index 0000000..14bd147 --- /dev/null +++ b/sdks/blockchain/uploader.go @@ -0,0 +1,179 @@ +package blockchain + +import ( + "fmt" + "gitlink.org.cn/cloudream/common/pkgs/types" + sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" + "time" +) + +type DataID int64 + +type Cluster struct { + DataID DataID `gorm:"column:dataID" json:"dataID"` + ClusterID schsdk.ClusterID `gorm:"column:clusterID" json:"clusterID"` + StorageID cdssdk.StorageID `gorm:"column:storageID" json:"storageID"` +} + +func (Cluster) TableName() string { + return "UploadedCluster" // 确保和数据库中的表名一致 +} + +type UploadedData struct { + ID DataID `gorm:"column:id;primaryKey" json:"id"` + Name string `gorm:"column:name" json:"name"` + DataType string `gorm:"column:dataType" json:"dataType"` + PackageID cdssdk.PackageID `gorm:"column:packageID" json:"packageID"` + JsonData string `gorm:"column:jsonData" json:"jsonData"` // JSON 数据字段 + UploadTime time.Time `gorm:"column:uploadTime" json:"uploadTime"` + UploadedCluster []Cluster `gorm:"foreignKey:dataID;references:id" json:"Clusters"` // 关联 Cluster 数据 +} + +type DataScheduleReq struct { + PackageID cdssdk.PackageID `json:"packageID"` + DataType string `json:"dataType"` + Clusters []Cluster `json:"clusters"` +} + +type codeRepository struct { + RepositoryName string + ClusterID ClusterID +} + +type DataScheduleResp struct { + Results []sch.DataScheduleResult `json:"results"` +} + +func (c *Client) DataSchedule(req DataScheduleReq) (*DataScheduleResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/jobSet/schedule") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[DataScheduleResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} + +type UploadReq struct { + Type string `json:"type"` + Source UploadSource `json:"source"` + Target UploadTarget `json:"target"` + StorageIDs []cdssdk.StorageID `json:"storageIDs"` +} + +type UploadSource interface { + Noop() +} + +var UploadSourceTypeUnion = types.NewTypeUnion[UploadSource]( + (*PackageSource)(nil), + (*UrlSource)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadSourceTypeUnion, "type") + +type PackageSource struct { + serder.Metadata `union:"packageSource"` + UploadSourceBase + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` +} + +type UrlSource struct { + serder.Metadata `union:"urlSource"` + UploadSourceBase + Type string `json:"type"` + Url string `json:"url"` +} + +type UploadSourceBase struct{} + +func (d *UploadSourceBase) Noop() {} + +type UploadTarget interface { + Noop() +} + +var UploadTargetTypeUnion = types.NewTypeUnion[UploadTarget]( + (*UrlTarget)(nil), + (*ApiTarget)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadTargetTypeUnion, "type") + +type UrlTarget struct { + serder.Metadata `union:"url"` + UploadTargetBase + Clusters []ClusterID `json:"clusters"` +} + +type ApiTarget struct { + serder.Metadata `union:"api"` + UploadTargetBase + Clusters []ClusterID `json:"clusters"` +} + +type UploadTargetBase struct{} + +func (d *UploadTargetBase) Noop() {} + +type UploadResp struct { + PackageID cdssdk.PackageID `json:"packageID"` + JsonData string `json:"jsonData"` +} + +func (c *Client) Upload(req UploadReq) (*UploadResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/data/upload") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[UploadResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index ea4d69c..450be9a 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -11,7 +11,12 @@ import ( ) type GetClusterInfoReq struct { - IDs []ClusterID `json:"ids"` + IDs []schsdk.ClusterID `json:"clusterIDs"` +} + +type GetClusterInfoResp struct { + Data []ClusterDetail `json:"data"` + TraceId string `json:"traceId"` } func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) { @@ -19,7 +24,8 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) if err != nil { return nil, err } - resp, err := http2.GetJSON(targetUrl, http2.RequestParam{Body: req}) + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{Body: req}) if err != nil { return nil, err } @@ -42,8 +48,8 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) } type CreateJobReq struct { - DataDistribute DataDistribute `json:"dataDistribute"` - Resources schsdk.JobResourcesInfo `json:"resources"` + JobResources schsdk.JobResources `json:"jobResources"` + DataDistribute DataDistribute `json:"dataDistribute"` } type DataDistribute struct { @@ -53,33 +59,33 @@ type DataDistribute struct { Model []ModelDistribute `json:"model"` } +type DataDetail struct { + ClusterID schsdk.ClusterID `json:"clusterID"` + JsonData string `json:"jsonData"` +} + type DatasetDistribute struct { DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` - Clusters []ClusterID `json:"clusters"` + Clusters []DataDetail `json:"clusters"` } type CodeDistribute struct { DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` - Clusters []ClusterID `json:"clusters"` + Clusters []DataDetail `json:"clusters"` } type ImageDistribute struct { DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` - Clusters []ClusterID `json:"clusters"` + Clusters []DataDetail `json:"clusters"` } type ModelDistribute struct { DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` - Clusters []ClusterID `json:"clusters"` -} - -type Cluster struct { - ClusterID ClusterID `json:"clusterID"` - StorageID cdssdk.StorageID `json:"storageID"` + Clusters []DataDetail `json:"clusters"` } type CreateJobResp struct { @@ -88,35 +94,40 @@ type CreateJobResp struct { } type ScheduleData struct { - DataType string `json:"dataType"` - PackageID cdssdk.PackageID `json:"packageID"` - StorageType string `json:"storageType"` - ClusterIDs []ClusterID `json:"clusterIDs"` + DataType string `json:"dataType"` + PackageID cdssdk.PackageID `json:"packageID"` + StorageType string `json:"storageType"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } -func (c *Client) CreateJob(req CreateJobReq) (CreateJobResp, error) { +func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/jobSet/submit") + if err != nil { + return nil, err + } -} + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } -type DataScheduleReq struct { - PackageID cdssdk.PackageID `json:"packageID"` - StorageType string `json:"storageType"` - Clusters []Cluster `json:"clusters"` -} + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[CreateJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } -type DataScheduleResp struct { - Results []DataScheduleResult `json:"results"` -} + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } -type DataScheduleResult struct { - ClusterID ClusterID `json:"clusterID"` - PackageID cdssdk.PackageID `json:"packageID"` - PackageFullPath string `json:"packageFullPath"` - Status bool `json:"status"` - Msg string `json:"msg"` -} + return nil, codeResp.ToError() + } -func (c *Client) DataSchedule(req DataScheduleReq) (DataScheduleResp, error) { + return nil, fmt.Errorf("unknow response content type: %s", contType) } @@ -125,12 +136,47 @@ type RunJobReq struct { ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` } +type DataScheduleResult struct { + Clusters DataDetail `json:"clusters"` + PackageID cdssdk.PackageID `json:"packageID"` + PackageFullPath string `json:"packageFullPath"` + Status bool `json:"status"` + Msg string `json:"msg"` +} + type DataScheduleResults struct { DataType string `json:"dataType"` Results []DataScheduleResult `json:"results"` } func (c *Client) RunJob(req RunJobReq) error { + targetUrl, err := url.JoinPath(c.baseURL, "/jobSet/submit") + if err != nil { + return err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + }) + if err != nil { + return err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return nil + } + + return codeResp.ToError() + } + + return fmt.Errorf("unknow response content type: %s", contType) } @@ -140,5 +186,28 @@ type CancelJobReq struct { } func (c *Client) CancelJob(req CancelJobReq) error { + targetUrl, err := url.JoinPath(c.baseURL, "/queryResources") + if err != nil { + return err + } + resp, err := http2.GetJSON(targetUrl, http2.RequestParam{Body: req}) + if err != nil { + return err + } + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + + var codeResp response[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return nil + } + + return codeResp.ToError() + } + return fmt.Errorf("unknow response content type: %s", contType) } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index e9bda19..466d560 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -2,6 +2,7 @@ package sch import ( "gitlink.org.cn/cloudream/common/pkgs/types" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" "gitlink.org.cn/cloudream/common/utils/serder" ) @@ -21,12 +22,26 @@ const ( MODEL = "model" ) -type ClusterID int64 type TaskID int64 type ClusterDetail struct { - ID ClusterID `json:"id"` - Resources []ResourceData `json:"resources"` + // 集群ID + ClusterId schsdk.ClusterID `json:"clusterID"` + // 集群功能类型：云算，智算，超算 + ClusterType string `json:"clusterType"` + // 集群地区：华东地区、华南地区、华北地区、华中地区、西南地区、西北地区、东北地区 + Region string `json:"region"` + // 资源类型 + Resources2 []ResourceData `json:"resources1,omitempty"` + //Resources []ResourceData `json:"resources"` + Resources []TmpResourceData `json:"resources"` +} + +type TmpResourceData struct { + Type ResourceType `json:"type"` + Name string `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` } type ResourceData interface { @@ -38,10 +53,13 @@ var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( (*NPUResourceData)(nil), (*GPUResourceData)(nil), (*MLUResourceData)(nil), + (*DCUResourceData)(nil), + (*GCUResourceData)(nil), + (*GPGPUResourceData)(nil), (*StorageResourceData)(nil), (*MemoryResourceData)(nil), ) -var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "name") +var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "type") type ResourceDataBase struct{} @@ -55,99 +73,84 @@ type UnitValue[T any] struct { type CPUResourceData struct { serder.Metadata `union:"CPU"` ResourceDataBase + Type string `json:"type"` Name ResourceType `json:"name"` Total UnitValue[int64] `json:"total"` Available UnitValue[int64] `json:"available"` } -func NewCPUResourceData(total UnitValue[int64], available UnitValue[int64]) *CPUResourceData { - return &CPUResourceData{ - Name: ResourceTypeCPU, - Total: total, - Available: available, - } -} - type NPUResourceData struct { serder.Metadata `union:"NPU"` ResourceDataBase + Type string `json:"type"` Name ResourceType `json:"name"` Total UnitValue[int64] `json:"total"` Available UnitValue[int64] `json:"available"` } -func NewNPUResourceData(total UnitValue[int64], available UnitValue[int64]) *NPUResourceData { - return &NPUResourceData{ - Name: ResourceTypeNPU, - Total: total, - Available: available, - } -} - type GPUResourceData struct { serder.Metadata `union:"GPU"` ResourceDataBase + Type string `json:"type"` Name ResourceType `json:"name"` Total UnitValue[int64] `json:"total"` Available UnitValue[int64] `json:"available"` } -func NewGPUResourceData(total UnitValue[int64], available UnitValue[int64]) *GPUResourceData { - return &GPUResourceData{ - Name: ResourceTypeGPU, - Total: total, - Available: available, - } -} - type MLUResourceData struct { serder.Metadata `union:"MLU"` ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type DCUResourceData struct { + serder.Metadata `union:"DCU"` + ResourceDataBase + Type string `json:"type"` Name ResourceType `json:"name"` Total UnitValue[int64] `json:"total"` Available UnitValue[int64] `json:"available"` } -func NewMLUResourceData(total UnitValue[int64], available UnitValue[int64]) *MLUResourceData { - return &MLUResourceData{ - Name: ResourceTypeMLU, - Total: total, - Available: available, - } +type GCUResourceData struct { + serder.Metadata `union:"GCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPGPUResourceData struct { + serder.Metadata `union:"ILUVATAR-GPGPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` } type StorageResourceData struct { serder.Metadata `union:"STORAGE"` ResourceDataBase + Type string `json:"type"` Name ResourceType `json:"name"` Total UnitValue[float64] `json:"total"` Available UnitValue[float64] `json:"available"` } -func NewStorageResourceData(total UnitValue[float64], available UnitValue[float64]) *StorageResourceData { - return &StorageResourceData{ - Name: ResourceTypeStorage, - Total: total, - Available: available, - } -} - type MemoryResourceData struct { serder.Metadata `union:"MEMORY"` ResourceDataBase + Type string `json:"type"` Name ResourceType `json:"name"` Total UnitValue[float64] `json:"total"` Available UnitValue[float64] `json:"available"` } -func NewMemoryResourceData(total UnitValue[float64], available UnitValue[float64]) *MemoryResourceData { - return &MemoryResourceData{ - Name: ResourceTypeMemory, - Total: total, - Available: available, - } -} - type ResourcePriority interface { Noop() } @@ -168,21 +171,31 @@ func (d *ResourcePriorityBase) Noop() {} type RegionPriority struct { serder.Metadata `union:"region"` ResourcePriorityBase + Type string `json:"type"` Options []string `json:"options"` } type ChipPriority struct { serder.Metadata `union:"chip"` ResourcePriorityBase + Type string `json:"type"` Options []string `json:"options"` } type BiasPriority struct { serder.Metadata `union:"bias"` ResourcePriorityBase + Type string `json:"type"` Options []string `json:"options"` } +type UploadParams struct { + DataType string `json:"dataType"` + DataName string `json:"dataName"` + UploadInfo UploadInfo `json:"uploadInfo"` + UploadPriority UploadPriority `json:"uploadPriority"` +} + type UploadInfo interface { Noop() } @@ -197,13 +210,16 @@ var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type") type LocalUploadInfo struct { serder.Metadata `union:"local"` UploadInfoBase + Type string `json:"type"` LocalPath string `json:"localPath"` } type RemoteUploadInfo struct { serder.Metadata `union:"url"` UploadInfoBase - Url string `json:"url"` + Type string `json:"type"` + Url string `json:"url"` + TargetClusters []schsdk.ClusterID `json:"targetClusters"` } type UploadInfoBase struct{} @@ -224,13 +240,15 @@ var _ = serder.UseTypeUnionInternallyTagged(&UploadPriorityTypeUnion, "type") type Preferences struct { serder.Metadata `union:"preference"` UploadPriorityBase + Type string `json:"type"` ResourcePriorities []ResourcePriority `json:"priorities"` } type SpecifyCluster struct { serder.Metadata `union:"specify"` UploadPriorityBase - Clusters []ClusterID `json:"clusters"` + Type string `json:"type"` + Clusters []schsdk.ClusterID `json:"clusters"` } type UploadPriorityBase struct{} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index b6821e7..f8bccf4 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -38,6 +38,8 @@ type ECSInstanceID string type NodeID int64 type Address string +type ClusterID string + type JobSetInfo struct { Jobs []JobInfo `json:"jobs"` } @@ -54,6 +56,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*UpdateMultiInstanceJobInfo)(nil), (*FinetuningJobInfo)(nil), (*DataPreprocessJobInfo)(nil), + (*PCMJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -76,6 +79,98 @@ type NormalJobInfo struct { ModelJobInfo ModelJobInfo `json:"modelJobInfo"` } +type PCMJobInfo struct { + serder.Metadata `union:"PCM"` + JobInfoBase + Type string `json:"type"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` +} + +type JobResources struct { + ScheduleStrategy string `json:"scheduleStrategy"` //任务分配策略：负载均衡、积分优先、随机分配等 + Clusters []ClusterInfo `json:"clusters"` +} + +type ClusterInfo struct { + ClusterID ClusterID `json:"clusterID"` + Resources []Resource `json:"resources"` + Runtime JobRuntimeInfo `json:"runtime"` +} + +type Resource struct { + Resource []JobResource `json:"resource"` +} + +type JobResource interface { + Noop() +} + +var JobResourceTypeUnion = types.NewTypeUnion[JobResource]( + (*CPU)(nil), + (*GPU)(nil), + (*NPU)(nil), + (*MLU)(nil), + (*DCU)(nil), + (*Memory)(nil), + (*PRICE)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type") + +type JobResourceBase struct{} + +func (d *JobResourceBase) Noop() {} + +type CPU struct { + serder.Metadata `union:"CPU"` + JobResourceBase + Type string `json:"type"` + Number int64 `json:"number"` +} + +type GPU struct { + serder.Metadata `union:"GPU"` + JobResourceBase + Type string `json:"type"` + Number int64 `json:"number"` +} + +type NPU struct { + serder.Metadata `union:"NPU"` + JobResourceBase + Type string `json:"type"` + Number int64 `json:"number"` +} + +type Memory struct { + serder.Metadata `union:"Memory"` + JobResourceBase + Type string `json:"type"` + Number int64 `json:"number"` +} + +type DCU struct { + serder.Metadata `union:"DCU"` + JobResourceBase + Type string `json:"type"` + Number int64 `json:"number"` +} + +type MLU struct { + serder.Metadata `union:"MLU"` + JobResourceBase + Type string `json:"type"` + Number int64 `json:"number"` +} + +type PRICE struct { + serder.Metadata `union:"PRICE"` + JobResourceBase + Type string `json:"type"` + Number int64 `json:"number"` +} + // FinetuningJobInfo 模型微调 type FinetuningJobInfo struct { serder.Metadata `union:"Finetuning"` @@ -154,6 +249,7 @@ type JobFilesInfo struct { Dataset JobFileInfo `json:"dataset"` Code JobFileInfo `json:"code"` Image JobFileInfo `json:"image"` + Model JobFileInfo `json:"model"` } type JobFileInfo interface { @@ -165,6 +261,7 @@ var FileInfoTypeUnion = types.NewTypeUnion[JobFileInfo]( (*LocalJobFileInfo)(nil), (*DataReturnJobFileInfo)(nil), (*ImageJobFileInfo)(nil), + (*BindingJobFileInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&FileInfoTypeUnion, "type") @@ -172,6 +269,13 @@ type JobFileInfoBase struct{} func (i *JobFileInfoBase) Noop() {} +type BindingJobFileInfo struct { + serder.Metadata `union:"Binding"` + JobFileInfoBase + Type string `json:"type"` + BindingID int64 `json:"bindingID"` +} + type PackageJobFileInfo struct { serder.Metadata `union:"Package"` JobFileInfoBase @@ -203,6 +307,7 @@ type ImageJobFileInfo struct { type JobRuntimeInfo struct { Command string `json:"command"` Envs []KVPair `json:"envs"` + Params []KVPair `json:"params"` } type KVPair struct { diff --git a/sdks/uploader/client.go b/sdks/uploader/client.go new file mode 100644 index 0000000..7b1f385 --- /dev/null +++ b/sdks/uploader/client.go @@ -0,0 +1,57 @@ +package uploadersdk + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/uploader/config.go b/sdks/uploader/config.go new file mode 100644 index 0000000..71cf23d --- /dev/null +++ b/sdks/uploader/config.go @@ -0,0 +1,5 @@ +package uploadersdk + +type Config struct { + URL string `json:"url"` +} diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go new file mode 100644 index 0000000..863cb1f --- /dev/null +++ b/sdks/uploader/models.go @@ -0,0 +1,28 @@ +package uploadersdk + +import cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + +type ClusterID string + +type BlockChain struct { + DataID DataID `gorm:"column:dataID" json:"dataID"` + BlockChainID string `gorm:"column:blockChainID" json:"blockChainID"` + FileName string `gorm:"column:fileName" json:"fileName"` + FileHash string `gorm:"column:fileHash" json:"fileHash"` + FileSize int64 `gorm:"column:fileSize" json:"fileSize"` +} + +func (BlockChain) TableName() string { + return "BlockChain" // 确保和数据库中的表名一致 +} + +type BindingData struct { + ID DataID `gorm:"column:ID" json:"ID"` + UserID cdssdk.UserID `gorm:"column:userID" json:"userID"` + BindingName string `gorm:"column:bindingName" json:"bindingName"` + BindingType string `gorm:"column:bindingType" json:"bindingType"` +} + +func (BindingData) TableName() string { + return "BindingData" // 确保和数据库中的表名一致 +} diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go new file mode 100644 index 0000000..8c607d4 --- /dev/null +++ b/sdks/uploader/uploader.go @@ -0,0 +1,182 @@ +package uploadersdk + +import ( + "fmt" + "gitlink.org.cn/cloudream/common/pkgs/types" + sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" + "time" +) + +type DataID int64 + +type Cluster struct { + DataID DataID `gorm:"column:dataID" json:"dataID"` + ClusterID schsdk.ClusterID `gorm:"column:clusterID" json:"clusterID"` + StorageID cdssdk.StorageID `gorm:"column:storageID" json:"storageID"` +} + +func (Cluster) TableName() string { + return "UploadedCluster" // 确保和数据库中的表名一致 +} + +type UploadedData struct { + ID DataID `gorm:"column:id;primaryKey" json:"id"` + UserID cdssdk.UserID `gorm:"column:userID" json:"userID"` + Name string `gorm:"column:name" json:"name"` + DataType string `gorm:"column:dataType" json:"dataType"` + PackageID cdssdk.PackageID `gorm:"column:packageID" json:"packageID"` + JsonData string `gorm:"column:jsonData" json:"jsonData"` // JSON 数据字段 + BindingID DataID `gorm:"column:bindingID" json:"bindingID"` + UploadTime time.Time `gorm:"column:uploadTime" json:"uploadTime"` + UploadedCluster []Cluster `gorm:"foreignKey:dataID;references:id" json:"clusters"` // 关联 Cluster 数据 + BlockChain []BlockChain `gorm:"foreignKey:dataID;references:id" json:"blockChains"` // 关联 BlockChain 数据 +} + +type DataScheduleReq struct { + PackageID cdssdk.PackageID `json:"packageID"` + DataType string `json:"dataType"` + Clusters []Cluster `json:"clusters"` +} + +type codeRepository struct { + RepositoryName string + ClusterID ClusterID +} + +type DataScheduleResp struct { + Results []sch.DataScheduleResult `json:"results"` +} + +func (c *Client) DataSchedule(req DataScheduleReq) (*DataScheduleResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/jobSet/schedule") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[DataScheduleResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} + +type UploadReq struct { + Type string `json:"type"` + Source UploadSource `json:"source"` + Target UploadTarget `json:"target"` + StorageIDs []cdssdk.StorageID `json:"storageIDs"` +} + +type UploadSource interface { + Noop() +} + +var UploadSourceTypeUnion = types.NewTypeUnion[UploadSource]( + (*PackageSource)(nil), + (*UrlSource)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadSourceTypeUnion, "type") + +type PackageSource struct { + serder.Metadata `union:"packageSource"` + UploadSourceBase + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` +} + +type UrlSource struct { + serder.Metadata `union:"urlSource"` + UploadSourceBase + Type string `json:"type"` + Url string `json:"url"` +} + +type UploadSourceBase struct{} + +func (d *UploadSourceBase) Noop() {} + +type UploadTarget interface { + Noop() +} + +var UploadTargetTypeUnion = types.NewTypeUnion[UploadTarget]( + (*UrlTarget)(nil), + (*ApiTarget)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadTargetTypeUnion, "type") + +type UrlTarget struct { + serder.Metadata `union:"url"` + UploadTargetBase + Clusters []ClusterID `json:"clusters"` +} + +type ApiTarget struct { + serder.Metadata `union:"api"` + UploadTargetBase + Clusters []ClusterID `json:"clusters"` +} + +type UploadTargetBase struct{} + +func (d *UploadTargetBase) Noop() {} + +type UploadResp struct { + PackageID cdssdk.PackageID `json:"packageID"` + JsonData string `json:"jsonData"` +} + +func (c *Client) Upload(req UploadReq) (*UploadResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/data/upload") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[UploadResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} diff --git a/utils/config/config.go b/utils/config/config.go index d1bdb89..3c4fe38 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -3,10 +3,9 @@ package config import ( "encoding/json" "fmt" + "github.com/imdario/mergo" "os" "path/filepath" - - "github.com/imdario/mergo" ) // Load 从本地文件读取配置，加载配置文件 @@ -32,6 +31,9 @@ func DefaultLoad(modeulName string, defCfg interface{}) error { // TODO 可以考虑根据环境变量读取不同的配置 // filepath.Join用于将多个路径组合成一个路径 configFilePath := filepath.Join(filepath.Dir(execPath), "..", "confs", fmt.Sprintf("%s.config.json", modeulName)) + + configFilePath = "D:\\Work\\Codes\\workspace\\workspace\\scheduler\\common\\assets\\confs\\middleware.json" + return Load(configFilePath, defCfg) } From 76941382183af10b224d372d9540149a21c46fb0 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 20 Dec 2024 16:17:47 +0800 Subject: [PATCH 003/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=A4=8D=E5=88=B6Pac?= =?UTF-8?q?kage=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/package.go | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index 6be57aa..a9418ff 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -221,6 +221,44 @@ func (c *PackageService) Delete(req PackageDelete) error { return fmt.Errorf("unknow response content type: %s", contType) } +const PackageClonePath = "/package/clone" + +type PackageClone struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` + Name string `json:"name" binding:"required"` +} + +type PackageCloneResp struct { + Package cdssdk.Package `json:"package"` +} + +func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) { + url, err := url.JoinPath(c.baseURL, PackageClonePath) + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(url, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + codeResp, err := ParseJSONResponse[response[PackageCloneResp]](resp) + if err != nil { + return nil, err + } + + if codeResp.Code == errorcode.OK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() +} + const PackageListBucketPackagesPath = "/package/listBucketPackages" type PackageListBucketPackages struct { From 111115389ae712de6980c6c4692745337e8cd006 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 26 Dec 2024 11:16:14 +0800 Subject: [PATCH 004/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=B0=83=E5=BA=A6?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/hub.go | 2 +- sdks/storage/cdsapi/package.go | 49 ++------------ sdks/storage/cdsapi/storage.go | 8 +-- sdks/storage/cdsapi/storage_test.go | 21 ------ sdks/storage/models.go | 2 +- utils/math2/range.go | 99 +++++++++++++++++++++++++++++ 6 files changed, 110 insertions(+), 71 deletions(-) create mode 100644 utils/math2/range.go diff --git a/sdks/storage/cdsapi/hub.go b/sdks/storage/cdsapi/hub.go index 09fa7bd..d1ed7b7 100644 --- a/sdks/storage/cdsapi/hub.go +++ b/sdks/storage/cdsapi/hub.go @@ -15,7 +15,7 @@ type HubGetHubsReq struct { } type HubGetHubsResp struct { - Hubs []cdssdk.Hub `json:"hubs"` + Hubs []*cdssdk.Hub `json:"hubs"` } func (c *Client) HubGetHubs(req HubGetHubsReq) (*HubGetHubsResp, error) { diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index a9418ff..5481370 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -135,15 +135,15 @@ type PackageCreateLoad struct { Files UploadObjectIterator `json:"-"` } type PackageCreateLoadInfo struct { - UserID cdssdk.UserID `json:"userID" binding:"required"` - BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` - Name string `json:"name" binding:"required"` - LoadTo []cdssdk.StorageID `json:"loadTo" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` + Name string `json:"name" binding:"required"` + LoadTo []cdssdk.StorageID `json:"loadTo"` + LoadToPath []string `json:"loadToPath"` } type PackageCreateLoadResp struct { - Package cdssdk.Package `json:"package"` - Objects []cdssdk.Object `json:"objects"` - LoadedDirs []string `json:"loadedDirs"` + Package cdssdk.Package `json:"package"` + Objects []cdssdk.Object `json:"objects"` } func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadResp, error) { @@ -329,38 +329,3 @@ func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*Pa return nil, codeResp.ToError() } - -const PackageGetLoadedStoragesPath = "/package/getLoadedStorages" - -type PackageGetLoadedStoragesReq struct { - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` -} - -type PackageGetLoadedStoragesResp struct { - StorageIDs []cdssdk.StorageID `json:"storageIDs"` -} - -func (c *PackageService) GetLoadedStorages(req PackageGetLoadedStoragesReq) (*PackageGetLoadedStoragesResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetLoadedStoragesPath) - if err != nil { - return nil, err - } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetLoadedStoragesResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() -} diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index 3050f0d..f45b234 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -17,13 +17,9 @@ type StorageLoadPackageReq struct { UserID cdssdk.UserID `json:"userID" binding:"required"` PackageID cdssdk.PackageID `json:"packageID" binding:"required"` StorageID cdssdk.StorageID `json:"storageID" binding:"required"` + RootPath string `json:"rootPath"` } -type StorageLoadPackageResp struct { - FullPath string `json:"fullPath"` // TODO 临时保留给中期测试的前端使用，后续会删除 - PackagePath string `json:"packagePath"` - LocalBase string `json:"localBase"` - RemoteBase string `json:"remoteBase"` -} +type StorageLoadPackageResp struct{} func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPackageResp, error) { url, err := url.JoinPath(c.baseURL, StorageLoadPackagePath) diff --git a/sdks/storage/cdsapi/storage_test.go b/sdks/storage/cdsapi/storage_test.go index e12f984..9f02704 100644 --- a/sdks/storage/cdsapi/storage_test.go +++ b/sdks/storage/cdsapi/storage_test.go @@ -245,24 +245,3 @@ func Test_Cache(t *testing.T) { So(err, ShouldBeNil) }) } - -func Test_GetHubInfos(t *testing.T) { - Convey("测试获取hub信息", t, func() { - cli := NewClient(&Config{ - URL: "http://localhost:7890", - }) - resp1, err := cli.Package().GetCachedStorages(PackageGetCachedStoragesReq{ - PackageID: 11, - UserID: 1, - }) - So(err, ShouldBeNil) - fmt.Printf("resp1: %v\n", resp1) - - resp2, err := cli.Package().GetLoadedStorages(PackageGetLoadedStoragesReq{ - PackageID: 11, - UserID: 1, - }) - So(err, ShouldBeNil) - fmt.Printf("resp2: %v\n", resp2) - }) -} diff --git a/sdks/storage/models.go b/sdks/storage/models.go index fc340b8..05df652 100644 --- a/sdks/storage/models.go +++ b/sdks/storage/models.go @@ -327,7 +327,7 @@ func (Bucket) TableName() string { type HubConnectivity struct { FromHubID HubID `gorm:"column:FromHubID; primaryKey; type:bigint" json:"fromHubID"` ToHubID HubID `gorm:"column:ToHubID; primaryKey; type:bigint" json:"ToHubID"` - Delay *float32 `gorm:"column:Delay; type:float" json:"delay"` + Latency *float32 `gorm:"column:Latency; type:float" json:"latency"` TestTime time.Time `gorm:"column:TestTime; type:datetime" json:"testTime"` } diff --git a/utils/math2/range.go b/utils/math2/range.go new file mode 100644 index 0000000..0f8ae2e --- /dev/null +++ b/utils/math2/range.go @@ -0,0 +1,99 @@ +package math2 + +type Range struct { + Offset int64 + Length *int64 +} + +// length为-1时Range.Length为nil +func NewRange(offset int64, length int64) Range { + if length == -1 { + return Range{Offset: offset, Length: nil} + } + return Range{Offset: offset, Length: &length} +} + +// 不包含end +func RangeFromStartEnd(start int64, end int64) Range { + length := end - start + return Range{Offset: start, Length: &length} +} + +// 给Length设置一个具体值 +func (r *Range) Fix(totalLen int64) { + len := totalLen - r.Offset + r.Length = &len +} + +// 如果Length为nil，则end为-1 +func (r *Range) ToStartEnd() (start int64, end int64) { + if r.Length == nil { + return r.Offset, -1 + } + + end = r.Offset + *r.Length + return r.Offset, end +} + +// 将范围限制在totalLen内。会同时设置Length的值 +func (r *Range) Clamp(totalLen int64) { + r.Offset = Min(r.Offset, totalLen) + if r.Length == nil { + len := totalLen - r.Offset + r.Length = &len + } else { + *r.Length = Min(*r.Length, totalLen-r.Offset) + } +} + +func (r *Range) Extend(other Range) { + newOffset := Min(r.Offset, other.Offset) + + if r.Length == nil { + r.Offset = newOffset + return + } + + if other.Length == nil { + r.Offset = newOffset + r.Length = nil + return + } + + otherEnd := other.Offset + *other.Length + rEnd := r.Offset + *r.Length + + newEnd := Max(otherEnd, rEnd) + r.Offset = newOffset + *r.Length = newEnd - newOffset +} + +func (r *Range) ExtendStart(start int64) { + r.Offset = Min(r.Offset, start) +} + +func (r *Range) ExtendEnd(end int64) { + if r.Length == nil { + return + } + + rEnd := r.Offset + *r.Length + newLen := Max(end, rEnd) - r.Offset + r.Length = &newLen +} + +func (r *Range) Equals(other Range) bool { + if r.Offset != other.Offset { + return false + } + + if r.Length == nil && other.Length == nil { + return true + } + + if r.Length == nil || other.Length == nil { + return false + } + + return *r.Length == *other.Length +} From 88947dc0237cef6ee2475eb563006221bcc99c9d Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 27 Dec 2024 10:18:27 +0800 Subject: [PATCH 005/155] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E6=94=AF=E6=8C=81=E8=B0=83=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 32d7243..b5d8d8c 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -70,10 +70,11 @@ type ObjectUpload struct { } type ObjectUploadInfo struct { - UserID cdssdk.UserID `json:"userID" binding:"required"` - PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - Affinity cdssdk.StorageID `json:"affinity"` - LoadTo []cdssdk.StorageID `json:"loadTo"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + Affinity cdssdk.StorageID `json:"affinity"` + LoadTo []cdssdk.StorageID `json:"loadTo"` + LoadToPath []string `json:"loadToPath"` } type UploadingObject struct { From fb0455250db567d3f8103bc729d3ce25a9d3190f Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 27 Dec 2024 15:20:23 +0800 Subject: [PATCH 006/155] =?UTF-8?q?=E4=BC=98=E5=8C=96sync2=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/sync2/bucket_pool.go | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 utils/sync2/bucket_pool.go diff --git a/utils/sync2/bucket_pool.go b/utils/sync2/bucket_pool.go new file mode 100644 index 0000000..2551a42 --- /dev/null +++ b/utils/sync2/bucket_pool.go @@ -0,0 +1,74 @@ +package sync2 + +import "sync" + +type BucketPool[T any] struct { + empty []T + filled []T + emptyCond *sync.Cond + filledCond *sync.Cond +} + +func NewBucketPool[T any]() *BucketPool[T] { + return &BucketPool[T]{ + emptyCond: sync.NewCond(&sync.Mutex{}), + filledCond: sync.NewCond(&sync.Mutex{}), + } +} + +func (p *BucketPool[T]) GetEmpty() (T, bool) { + p.emptyCond.L.Lock() + defer p.emptyCond.L.Unlock() + + if len(p.empty) == 0 { + p.emptyCond.Wait() + } + + if len(p.empty) == 0 { + var t T + return t, false + } + + t := p.empty[0] + p.empty = p.empty[1:] + return t, true +} + +func (p *BucketPool[T]) PutEmpty(t T) { + p.emptyCond.L.Lock() + defer p.emptyCond.L.Unlock() + + p.empty = append(p.empty, t) + p.emptyCond.Signal() +} + +func (p *BucketPool[T]) GetFilled() (T, bool) { + p.filledCond.L.Lock() + defer p.filledCond.L.Unlock() + + if len(p.filled) == 0 { + p.filledCond.Wait() + } + + if len(p.filled) == 0 { + var t T + return t, false + } + + t := p.filled[0] + p.filled = p.filled[1:] + return t, true +} + +func (p *BucketPool[T]) PutFilled(t T) { + p.filledCond.L.Lock() + defer p.filledCond.L.Unlock() + + p.filled = append(p.filled, t) + p.filledCond.Signal() +} + +func (p *BucketPool[T]) WakeUpAll() { + p.emptyCond.Broadcast() + p.filledCond.Broadcast() +} From 675cabab6034cf709d1cba24637e9531cbef573a Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 27 Dec 2024 15:44:32 +0800 Subject: [PATCH 007/155] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E7=9A=84Range=E7=BB=93=E6=9E=84=E4=BD=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/ioswitch/exec/driver.go | 3 +- pkgs/ioswitch/exec/utils.go | 88 -------------------------------- pkgs/ioswitch/plan/ops/driver.go | 3 +- 3 files changed, 4 insertions(+), 90 deletions(-) diff --git a/pkgs/ioswitch/exec/driver.go b/pkgs/ioswitch/exec/driver.go index 5c89ade..433b7c3 100644 --- a/pkgs/ioswitch/exec/driver.go +++ b/pkgs/ioswitch/exec/driver.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/go-multierror" "gitlink.org.cn/cloudream/common/pkgs/future" "gitlink.org.cn/cloudream/common/utils/io2" + "gitlink.org.cn/cloudream/common/utils/math2" ) type Driver struct { @@ -104,7 +105,7 @@ func (e *Driver) execute() { type DriverWriteStream struct { ID VarID - RangeHint *Range + RangeHint *math2.Range } type DriverReadStream struct { diff --git a/pkgs/ioswitch/exec/utils.go b/pkgs/ioswitch/exec/utils.go index 64b00c9..5186943 100644 --- a/pkgs/ioswitch/exec/utils.go +++ b/pkgs/ioswitch/exec/utils.go @@ -2,96 +2,8 @@ package exec import ( "github.com/google/uuid" - "gitlink.org.cn/cloudream/common/utils/math2" ) func genRandomPlanID() PlanID { return PlanID(uuid.NewString()) } - -type Range struct { - Offset int64 - Length *int64 -} - -func NewRange(offset int64, length int64) Range { - return Range{Offset: offset, Length: &length} -} - -func (r *Range) Extend(other Range) { - newOffset := math2.Min(r.Offset, other.Offset) - - if r.Length == nil { - r.Offset = newOffset - return - } - - if other.Length == nil { - r.Offset = newOffset - r.Length = nil - return - } - - otherEnd := other.Offset + *other.Length - rEnd := r.Offset + *r.Length - - newEnd := math2.Max(otherEnd, rEnd) - r.Offset = newOffset - *r.Length = newEnd - newOffset -} - -func (r *Range) ExtendStart(start int64) { - r.Offset = math2.Min(r.Offset, start) -} - -func (r *Range) ExtendEnd(end int64) { - if r.Length == nil { - return - } - - rEnd := r.Offset + *r.Length - newLen := math2.Max(end, rEnd) - r.Offset - r.Length = &newLen -} - -func (r *Range) Fix(maxLength int64) { - if r.Length != nil { - return - } - - len := maxLength - r.Offset - r.Length = &len -} - -func (r *Range) ToStartEnd(maxLen int64) (start int64, end int64) { - if r.Length == nil { - return r.Offset, maxLen - } - - end = r.Offset + *r.Length - return r.Offset, end -} - -func (r *Range) ClampLength(maxLen int64) { - if r.Length == nil { - return - } - - *r.Length = math2.Min(*r.Length, maxLen-r.Offset) -} - -func (r *Range) Equals(other Range) bool { - if r.Offset != other.Offset { - return false - } - - if r.Length == nil && other.Length == nil { - return true - } - - if r.Length == nil || other.Length == nil { - return false - } - - return *r.Length == *other.Length -} diff --git a/pkgs/ioswitch/plan/ops/driver.go b/pkgs/ioswitch/plan/ops/driver.go index da3c62f..66f4395 100644 --- a/pkgs/ioswitch/plan/ops/driver.go +++ b/pkgs/ioswitch/plan/ops/driver.go @@ -3,6 +3,7 @@ package ops import ( "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" + "gitlink.org.cn/cloudream/common/utils/math2" ) type FromDriverNode struct { @@ -40,7 +41,7 @@ func (t *FromDriverNode) GenerateOp() (exec.Op, error) { type ToDriverNode struct { dag.NodeBase Handle *exec.DriverReadStream - Range exec.Range + Range math2.Range } func (b *GraphNodeBuilder) NewToDriver(handle *exec.DriverReadStream) *ToDriverNode { From 1430cd891edff8fa900d3675807231e6da7d9374 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 30 Dec 2024 15:43:47 +0800 Subject: [PATCH 008/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A0=B9=E6=8D=AEID?= =?UTF-8?q?=E8=8E=B7=E5=8F=96Object=E7=9A=84=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index b5d8d8c..dfc1036 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -62,6 +62,41 @@ func (c *ObjectService) List(req ObjectList) (*ObjectListResp, error) { return nil, jsonResp.ToError() } +const ObjectListByIDsPath = "/object/listByIDs" + +type ObjectListByIDs struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + ObjectIDs []cdssdk.ObjectID `json:"objectIDs" binding:"required"` +} +type ObjectListByIDsResp struct { + Objects []*cdssdk.Object `json:"object"` // 与ObjectIDs一一对应，如果某个ID不存在，则对应位置为nil +} + +func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) { + url, err := url.JoinPath(c.baseURL, ObjectListByIDsPath) + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(url, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + jsonResp, err := ParseJSONResponse[response[ObjectListByIDsResp]](resp) + if err != nil { + return nil, err + } + + if jsonResp.Code == errorcode.OK { + return &jsonResp.Data, nil + } + + return nil, jsonResp.ToError() +} + const ObjectUploadPath = "/object/upload" type ObjectUpload struct { From a6c080409285ba765f707f10230581bed8b72c83 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 31 Dec 2024 09:14:04 +0800 Subject: [PATCH 009/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B0=83=E5=BA=A6?= =?UTF-8?q?=E4=B8=AD=E9=97=B4=E4=BB=B6=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 8 +++---- sdks/pcmscheduler/models.go | 18 +++++++++++++- sdks/scheduler/models.go | 3 +++ sdks/uploader/models.go | 25 +++++++++++++++----- sdks/uploader/uploader.go | 47 +++++++++++++++++++++++-------------- 5 files changed, 72 insertions(+), 29 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 450be9a..e5b69b0 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -65,25 +65,25 @@ type DataDetail struct { } type DatasetDistribute struct { - DataName string `json:"dataName"` + //DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } type CodeDistribute struct { - DataName string `json:"dataName"` + //DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } type ImageDistribute struct { - DataName string `json:"dataName"` + //DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } type ModelDistribute struct { - DataName string `json:"dataName"` + //DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 466d560..3f7dc3e 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -3,6 +3,7 @@ package sch import ( "gitlink.org.cn/cloudream/common/pkgs/types" schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/serder" ) @@ -20,6 +21,10 @@ const ( DATASET = "dataset" IMAGE = "image" MODEL = "model" + + OrderByName = "name" + OrderBySize = "size" + OrderByTime = "time" ) type TaskID int64 @@ -191,7 +196,6 @@ type BiasPriority struct { type UploadParams struct { DataType string `json:"dataType"` - DataName string `json:"dataName"` UploadInfo UploadInfo `json:"uploadInfo"` UploadPriority UploadPriority `json:"uploadPriority"` } @@ -219,6 +223,7 @@ type RemoteUploadInfo struct { UploadInfoBase Type string `json:"type"` Url string `json:"url"` + DataName string `json:"dataName"` TargetClusters []schsdk.ClusterID `json:"targetClusters"` } @@ -254,3 +259,14 @@ type SpecifyCluster struct { type UploadPriorityBase struct{} func (d *UploadPriorityBase) Noop() {} + +type QueryData struct { + DataType string `json:"dataType" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + //ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Path string `json:"path"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + CurrentPage int `json:"currentPage" binding:"required"` + PageSize int `json:"pageSize" binding:"required"` + OrderBy string `json:"orderBy" binding:"required"` +} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index f8bccf4..e37db88 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -18,6 +18,9 @@ const ( FileInfoTypeResource = "Resource" FileInfoTypeImage = "Image" + FILE = "file" + FOLDER = "folder" + MemoryUtilization = "MemoryUtilization" GPUUtilization = "GPUUtilization" CPUUtilization = "CPUUtilization" diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 863cb1f..bc4f74d 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -1,15 +1,18 @@ package uploadersdk -import cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) type ClusterID string type BlockChain struct { - DataID DataID `gorm:"column:dataID" json:"dataID"` - BlockChainID string `gorm:"column:blockChainID" json:"blockChainID"` - FileName string `gorm:"column:fileName" json:"fileName"` - FileHash string `gorm:"column:fileHash" json:"fileHash"` - FileSize int64 `gorm:"column:fileSize" json:"fileSize"` + ObjectID cdssdk.ObjectID `gorm:"column:object_id" json:"objectID"` + BlockChainID string `gorm:"column:blockChain_id" json:"blockChainID"` + //FileName string `gorm:"column:file_name" json:"fileName"` + //FileHash string `gorm:"column:file_hash" json:"fileHash"` + //FileSize int64 `gorm:"column:file_size" json:"fileSize"` } func (BlockChain) TableName() string { @@ -26,3 +29,13 @@ type BindingData struct { func (BindingData) TableName() string { return "BindingData" // 确保和数据库中的表名一致 } + +type Folder struct { + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + Path string `gorm:"column:path_name" json:"path"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` +} + +func (Folder) TableName() string { + return "folders" +} diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go index 8c607d4..254240d 100644 --- a/sdks/uploader/uploader.go +++ b/sdks/uploader/uploader.go @@ -10,32 +10,42 @@ import ( "gitlink.org.cn/cloudream/common/utils/serder" "net/url" "strings" - "time" ) type DataID int64 +type FolderID int64 type Cluster struct { - DataID DataID `gorm:"column:dataID" json:"dataID"` - ClusterID schsdk.ClusterID `gorm:"column:clusterID" json:"clusterID"` - StorageID cdssdk.StorageID `gorm:"column:storageID" json:"storageID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"dataID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + StorageID cdssdk.StorageID `gorm:"column:storage_id" json:"storageID"` } func (Cluster) TableName() string { - return "UploadedCluster" // 确保和数据库中的表名一致 + return "uploadedCluster" // 确保和数据库中的表名一致 } -type UploadedData struct { - ID DataID `gorm:"column:id;primaryKey" json:"id"` - UserID cdssdk.UserID `gorm:"column:userID" json:"userID"` - Name string `gorm:"column:name" json:"name"` - DataType string `gorm:"column:dataType" json:"dataType"` - PackageID cdssdk.PackageID `gorm:"column:packageID" json:"packageID"` - JsonData string `gorm:"column:jsonData" json:"jsonData"` // JSON 数据字段 - BindingID DataID `gorm:"column:bindingID" json:"bindingID"` - UploadTime time.Time `gorm:"column:uploadTime" json:"uploadTime"` - UploadedCluster []Cluster `gorm:"foreignKey:dataID;references:id" json:"clusters"` // 关联 Cluster 数据 - BlockChain []BlockChain `gorm:"foreignKey:dataID;references:id" json:"blockChains"` // 关联 BlockChain 数据 +type Package struct { + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + DataType string `gorm:"column:data_type" json:"dataType"` + JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` + UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` + //UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 + //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 +} + +type PackageDAO struct { + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + DataType string `gorm:"column:data_type" json:"dataType"` + JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 } type DataScheduleReq struct { @@ -147,8 +157,9 @@ type UploadTargetBase struct{} func (d *UploadTargetBase) Noop() {} type UploadResp struct { - PackageID cdssdk.PackageID `json:"packageID"` - JsonData string `json:"jsonData"` + PackageID cdssdk.PackageID `json:"packageID"` + ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` + JsonData string `json:"jsonData"` } func (c *Client) Upload(req UploadReq) (*UploadResp, error) { From 102bf9e82bb0dbc890b58cb8644e649b4951fec4 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 31 Dec 2024 10:17:12 +0800 Subject: [PATCH 010/155] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/config/config.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 3c4fe38..d0f930b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -3,9 +3,10 @@ package config import ( "encoding/json" "fmt" - "github.com/imdario/mergo" "os" "path/filepath" + + "github.com/imdario/mergo" ) // Load 从本地文件读取配置，加载配置文件 @@ -32,8 +33,6 @@ func DefaultLoad(modeulName string, defCfg interface{}) error { // filepath.Join用于将多个路径组合成一个路径 configFilePath := filepath.Join(filepath.Dir(execPath), "..", "confs", fmt.Sprintf("%s.config.json", modeulName)) - configFilePath = "D:\\Work\\Codes\\workspace\\workspace\\scheduler\\common\\assets\\confs\\middleware.json" - return Load(configFilePath, defCfg) } From cd5331ecd55ae6c2228e7280847ec7d3766a8853 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 31 Dec 2024 15:15:01 +0800 Subject: [PATCH 011/155] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E6=9C=8D=E5=8A=A1=E7=9B=B4=E4=BC=A0Feature?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/storage_feature.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sdks/storage/storage_feature.go b/sdks/storage/storage_feature.go index 0e150cb..2c6e1d8 100644 --- a/sdks/storage/storage_feature.go +++ b/sdks/storage/storage_feature.go @@ -78,3 +78,13 @@ func (f *InternalServerlessCallFeature) GetFeatureType() string { func (f *InternalServerlessCallFeature) String() string { return "InternalServerlessCall" } + +// 存储服务之间直传文件 +type S2SFeature struct { + serder.Metadata `union:"S2S"` + Type string `json:"type"` +} + +func (f *S2SFeature) GetFeatureType() string { + return "S2S" +} From 8fdf1f44f0a8a72f5a7c935097a635ce184d92bc Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 2 Jan 2025 09:56:52 +0800 Subject: [PATCH 012/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=A4=8D=E5=88=B6Obj?= =?UTF-8?q?ect=E7=9A=84=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index dfc1036..03774d2 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -456,6 +456,48 @@ func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error { return jsonResp.ToError() } +const ObjectClonePath = "/object/clone" + +type ObjectClone struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + Clonings []CloningObject `json:"clonings" binding:"required"` +} + +type CloningObject struct { + ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"` + NewPath string `json:"newPath" binding:"required"` + NewPackageID cdssdk.PackageID `json:"newPackageID" binding:"required"` +} + +type ObjectCloneResp struct { + Objects []*cdssdk.Object `json:"objects"` +} + +func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) { + url, err := url.JoinPath(c.baseURL, ObjectClonePath) + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(url, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + jsonResp, err := ParseJSONResponse[response[ObjectCloneResp]](resp) + if err != nil { + return nil, err + } + + if jsonResp.Code == errorcode.OK { + return &jsonResp.Data, nil + } + + return nil, jsonResp.ToError() +} + const ObjectGetPackageObjectsPath = "/object/getPackageObjects" type ObjectGetPackageObjects struct { From 8fed4c9e74ab03446761fb17f708ab22a64cf5c0 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 10 Jan 2025 10:17:31 +0800 Subject: [PATCH 013/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/ioswitch/dag/node.go | 4 ++++ pkgs/ioswitch/dag/var.go | 5 ----- pkgs/ioswitch/plan/ops/store.go | 39 +++++++++++++++++++++++++++++++++ sdks/storage/storage_feature.go | 13 +++++++---- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/pkgs/ioswitch/dag/node.go b/pkgs/ioswitch/dag/node.go index 60f8ea9..e8f0f1d 100644 --- a/pkgs/ioswitch/dag/node.go +++ b/pkgs/ioswitch/dag/node.go @@ -475,6 +475,10 @@ func (s ValueOutputSlot) Var() *ValueVar { return s.Node.OutputValues().Get(s.Index) } +func (s ValueOutputSlot) ToSlot(slot ValueInputSlot) { + s.Var().To(slot.Node, slot.Index) +} + type ValueInputSlot struct { Node Node Index int diff --git a/pkgs/ioswitch/dag/var.go b/pkgs/ioswitch/dag/var.go index 1a2f49c..27168a3 100644 --- a/pkgs/ioswitch/dag/var.go +++ b/pkgs/ioswitch/dag/var.go @@ -64,11 +64,6 @@ func (v *ValueVar) To(to Node, slotIdx int) { to.InputValues().Slots.Set(slotIdx, v) } -func (v *ValueVar) ToSlot(slot ValueInputSlot) { - v.Dst.Add(slot.Node) - slot.Node.InputValues().Slots.Set(slot.Index, v) -} - func (v *ValueVar) NotTo(node Node) { v.Dst.Remove(node) node.InputValues().Slots.Clear(v) diff --git a/pkgs/ioswitch/plan/ops/store.go b/pkgs/ioswitch/plan/ops/store.go index 977af42..4353dc7 100644 --- a/pkgs/ioswitch/plan/ops/store.go +++ b/pkgs/ioswitch/plan/ops/store.go @@ -26,6 +26,20 @@ func (o *Store) String() string { return fmt.Sprintf("Store %v: %v", o.Key, o.Var) } +type StoreConst struct { + Key string + Value exec.VarValue +} + +func (o *StoreConst) Execute(ctx *exec.ExecContext, e *exec.Executor) error { + e.Store(o.Key, o.Value) + return nil +} + +func (o *StoreConst) String() string { + return fmt.Sprintf("StoreConst %v: %v", o.Key, o.Value) +} + type StoreNode struct { dag.NodeBase Key string @@ -53,3 +67,28 @@ func (t *StoreNode) GenerateOp() (exec.Op, error) { // func (t *StoreType) String() string { // return fmt.Sprintf("Store[%s]%v%v", t.StoreKey, formatStreamIO(node), formatValueIO(node)) // } + +type StoreConstNode struct { + dag.NodeBase + Key string + Value exec.VarValue +} + +func (b *GraphNodeBuilder) NewStoreConst(key string, value exec.VarValue) *StoreConstNode { + node := &StoreConstNode{ + Key: key, + Value: value, + } + b.AddNode(node) + return node +} + +func (t *StoreConstNode) GenerateOp() (exec.Op, error) { + return &StoreConst{ + Key: t.Key, + Value: t.Value, + }, nil +} + +// func (t *StoreConstType) String() string { +// return fmt.Sprintf("StoreConst[%s]%v%v", t.StoreKey, formatStreamIO(node), formatValueIO(node)) diff --git a/sdks/storage/storage_feature.go b/sdks/storage/storage_feature.go index 2c6e1d8..5cf85b7 100644 --- a/sdks/storage/storage_feature.go +++ b/sdks/storage/storage_feature.go @@ -80,11 +80,16 @@ func (f *InternalServerlessCallFeature) String() string { } // 存储服务之间直传文件 -type S2SFeature struct { - serder.Metadata `union:"S2S"` +type S2STransferFeature struct { + serder.Metadata `union:"S2STransfer"` Type string `json:"type"` + TempDir string `json:"tempDir"` // 临时文件存放目录 } -func (f *S2SFeature) GetFeatureType() string { - return "S2S" +func (f *S2STransferFeature) GetFeatureType() string { + return "S2STransfer" +} + +func (f *S2STransferFeature) String() string { + return "S2STransfer" } From 4e81487b0f9cbd924f9313982accb6195c3dd81b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 10 Jan 2025 15:35:16 +0800 Subject: [PATCH 014/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=90=8D=E5=AD=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 12 ++++++------ sdks/storage/cdsapi/package.go | 12 ++++++------ sdks/storage/cdsapi/storage_test.go | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 03774d2..e3486fb 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -25,20 +25,20 @@ func (c *Client) Object() *ObjectService { } } -const ObjectListPath = "/object/list" +const ObjectListPathByPath = "/object/listByPath" -type ObjectList struct { +type ObjectListByPath struct { UserID cdssdk.UserID `form:"userID" binding:"required"` PackageID cdssdk.PackageID `form:"packageID" binding:"required"` Path string `form:"path"` // 允许为空字符串 IsPrefix bool `form:"isPrefix"` } -type ObjectListResp struct { +type ObjectListByPathResp struct { Objects []cdssdk.Object `json:"objects"` } -func (c *ObjectService) List(req ObjectList) (*ObjectListResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectListPath) +func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { + url, err := url.JoinPath(c.baseURL, ObjectListPathByPath) if err != nil { return nil, err } @@ -50,7 +50,7 @@ func (c *ObjectService) List(req ObjectList) (*ObjectListResp, error) { return nil, err } - jsonResp, err := ParseJSONResponse[response[ObjectListResp]](resp) + jsonResp, err := ParseJSONResponse[response[ObjectListByPathResp]](resp) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index 5481370..e72ae3e 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -55,19 +55,19 @@ func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { return nil, codeResp.ToError() } -const PackageGetByNamePath = "/package/getByName" +const PackageGetByFullNamePath = "/package/getByFullName" -type PackageGetByName struct { +type PackageGetByFullName struct { UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` BucketName string `form:"bucketName" json:"bucketName" binding:"required"` PackageName string `form:"packageName" json:"packageName" binding:"required"` } -type PackageGetByNameResp struct { +type PackageGetByFullNameResp struct { Package cdssdk.Package `json:"package"` } -func (c *PackageService) GetByName(req PackageGetByName) (*PackageGetByNameResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetByNamePath) +func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { + url, err := url.JoinPath(c.baseURL, PackageGetByFullNamePath) if err != nil { return nil, err } @@ -79,7 +79,7 @@ func (c *PackageService) GetByName(req PackageGetByName) (*PackageGetByNameResp, return nil, err } - codeResp, err := ParseJSONResponse[response[PackageGetByNameResp]](resp) + codeResp, err := ParseJSONResponse[response[PackageGetByFullNameResp]](resp) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/storage_test.go b/sdks/storage/cdsapi/storage_test.go index 9f02704..edc1994 100644 --- a/sdks/storage/cdsapi/storage_test.go +++ b/sdks/storage/cdsapi/storage_test.go @@ -131,7 +131,7 @@ func Test_ObjectList(t *testing.T) { URL: "http://localhost:7890", }) - resp, err := cli.Object().List(ObjectList{ + resp, err := cli.Object().ListByPath(ObjectListByPath{ UserID: 1, PackageID: 10, Path: "100x100K/zexema", From eb443e5169eee11a95e0f861cf26a9b96b996c10 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 13 Jan 2025 11:30:43 +0800 Subject: [PATCH 015/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0aws=E9=89=B4=E6=9D=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consts/errorcode/error_code.go | 1 + 1 file changed, 1 insertion(+) diff --git a/consts/errorcode/error_code.go b/consts/errorcode/error_code.go index 984ca9c..8dffeb3 100644 --- a/consts/errorcode/error_code.go +++ b/consts/errorcode/error_code.go @@ -7,4 +7,5 @@ const ( DataExists = "DataExists" BadArgument = "BadArgument" TaskNotFound = "TaskNotFound" + Unauthorized = "Unauthorized" ) From d60cce250e5cfeecda738995696cc537ecb33a1b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 15 Jan 2025 14:37:21 +0800 Subject: [PATCH 016/155] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/storage.go | 1 + sdks/storage/storage_feature.go | 1 + 2 files changed, 2 insertions(+) diff --git a/sdks/storage/storage.go b/sdks/storage/storage.go index 0d1fcd8..5986f2f 100644 --- a/sdks/storage/storage.go +++ b/sdks/storage/storage.go @@ -84,6 +84,7 @@ type OBSType struct { SK string `json:"secretAccessKey"` Endpoint string `json:"endpoint"` Bucket string `json:"bucket"` + ProjectID string `json:"projectID"` } func (a *OBSType) GetStorageType() string { diff --git a/sdks/storage/storage_feature.go b/sdks/storage/storage_feature.go index 5cf85b7..db6f9f2 100644 --- a/sdks/storage/storage_feature.go +++ b/sdks/storage/storage_feature.go @@ -17,6 +17,7 @@ var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Storage (*BypassWriteFeature)(nil), (*MultipartUploadFeature)(nil), (*InternalServerlessCallFeature)(nil), + (*S2STransferFeature)(nil), )), "type") type TempStore struct { From 3bd0a8a3aead88e5fc4a314473ace5a7ca2e5ec9 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 15 Jan 2025 17:07:57 +0800 Subject: [PATCH 017/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/ioswitch/plan/{generate.go => compile.go} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename pkgs/ioswitch/plan/{generate.go => compile.go} (98%) diff --git a/pkgs/ioswitch/plan/generate.go b/pkgs/ioswitch/plan/compile.go similarity index 98% rename from pkgs/ioswitch/plan/generate.go rename to pkgs/ioswitch/plan/compile.go index addeaed..6e75abe 100644 --- a/pkgs/ioswitch/plan/generate.go +++ b/pkgs/ioswitch/plan/compile.go @@ -6,7 +6,7 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops" ) -func Generate(graph *dag.Graph, planBld *exec.PlanBuilder) error { +func Compile(graph *dag.Graph, planBld *exec.PlanBuilder) error { myGraph := &ops.GraphNodeBuilder{graph} generateSend(myGraph) return buildPlan(graph, planBld) From 08ce051df0b43515ba93b01200dfbb5e73403e9f Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 16 Jan 2025 16:26:41 +0800 Subject: [PATCH 018/155] =?UTF-8?q?=E4=B8=AD=E9=97=B4=E4=BB=B6=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E6=9D=83=E9=99=90=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 74 +++++++++++++- sdks/blockchain/client.go | 10 +- sdks/blockchain/config.go | 7 +- sdks/blockchain/uploader.go | 179 --------------------------------- sdks/pcmscheduler/access.go | 41 ++++++++ sdks/pcmscheduler/jobset.go | 39 +++++--- sdks/pcmscheduler/models.go | 181 ++++++++++++++++++++++++++++++++-- sdks/scheduler/models.go | 20 ++-- sdks/uploader/models.go | 37 +++++-- sdks/uploader/uploader.go | 92 +++++++++++++---- 10 files changed, 434 insertions(+), 246 deletions(-) delete mode 100644 sdks/blockchain/uploader.go create mode 100644 sdks/pcmscheduler/access.go diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go index 9a51461..b85b859 100644 --- a/sdks/blockchain/blockchain.go +++ b/sdks/blockchain/blockchain.go @@ -1,8 +1,11 @@ package blockchain import ( + "fmt" "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" "net/url" + "strings" ) type InvokeReq struct { @@ -14,15 +17,21 @@ type InvokeReq struct { } func (c *Client) BlockChainInvoke(req InvokeReq) error { - targetUrl, err := url.JoinPath(c.baseURL, "/contract/invoke") + targetUrl, err := url.JoinPath(c.baseURL, "/jcc-bcos/contract/invoke") + if err != nil { + return err + } + + token, err := c.getToken() if err != nil { return err } header := make(map[string]string) header["Content-Type"] = http2.ContentTypeJSON + header["Authorization"] = token - _, err = http2.PostJSON(targetUrl, http2.RequestParam{ + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, Header: header, }) @@ -30,5 +39,66 @@ func (c *Client) BlockChainInvoke(req InvokeReq) error { return err } + var codeResp response[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code != ResponseCodeOK { + return codeResp.ToError() + } + return nil } + +type TokenReq struct { + Username string `json:"username"` + Password string `json:"password"` +} + +type TokenResp struct { + RoleID string `json:"roleId"` + Token string `json:"token"` +} + +func (c *Client) getToken() (string, error) { + targetUrl, err := url.JoinPath(c.loginUrl, "/jcc-admin/admin/login") + if err != nil { + return "", err + } + + req := TokenReq{ + Username: c.userName, + Password: c.password, + } + + header := make(map[string]string) + header["User-Agent"] = "Apifox/1.0.0 (https://apifox.com)" + header["Content-Type"] = "application/json" + header["Accept"] = "*/*" + header["Host"] = "dev.jointcloud.net" + header["Connection"] = "keep-alive" + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: header, + }) + if err != nil { + return "", err + } + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + + var codeResp response[TokenResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return "", fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return "Bearer " + codeResp.Data.Token, nil + } + + return "", codeResp.ToError() + } + return "", fmt.Errorf("error: %w", err) +} diff --git a/sdks/blockchain/client.go b/sdks/blockchain/client.go index 7121aa0..7b4b5af 100644 --- a/sdks/blockchain/client.go +++ b/sdks/blockchain/client.go @@ -24,12 +24,18 @@ func (r *response[T]) ToError() *sdks.CodeMessageError { } type Client struct { - baseURL string + baseURL string + loginUrl string + userName string + password string } func NewClient(cfg *Config) *Client { return &Client{ - baseURL: cfg.URL, + baseURL: cfg.URL, + loginUrl: cfg.LoginUrl, + userName: cfg.UserName, + password: cfg.Password, } } diff --git a/sdks/blockchain/config.go b/sdks/blockchain/config.go index 59254f5..a3e02f4 100644 --- a/sdks/blockchain/config.go +++ b/sdks/blockchain/config.go @@ -4,6 +4,9 @@ type Config struct { URL string `json:"url"` ContractAddress string `json:"contractAddress"` FunctionName string `json:"functionName"` - MemberName string `json:"memberName"` - Type string `json:"type"` + LoginUrl string `json:"loginUrl"` + UserName string `json:"userName"` + Password string `json:"password"` + //MemberName string `json:"memberName"` + //Type string `json:"type"` } diff --git a/sdks/blockchain/uploader.go b/sdks/blockchain/uploader.go deleted file mode 100644 index 14bd147..0000000 --- a/sdks/blockchain/uploader.go +++ /dev/null @@ -1,179 +0,0 @@ -package blockchain - -import ( - "fmt" - "gitlink.org.cn/cloudream/common/pkgs/types" - sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" - schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" - "gitlink.org.cn/cloudream/common/utils/serder" - "net/url" - "strings" - "time" -) - -type DataID int64 - -type Cluster struct { - DataID DataID `gorm:"column:dataID" json:"dataID"` - ClusterID schsdk.ClusterID `gorm:"column:clusterID" json:"clusterID"` - StorageID cdssdk.StorageID `gorm:"column:storageID" json:"storageID"` -} - -func (Cluster) TableName() string { - return "UploadedCluster" // 确保和数据库中的表名一致 -} - -type UploadedData struct { - ID DataID `gorm:"column:id;primaryKey" json:"id"` - Name string `gorm:"column:name" json:"name"` - DataType string `gorm:"column:dataType" json:"dataType"` - PackageID cdssdk.PackageID `gorm:"column:packageID" json:"packageID"` - JsonData string `gorm:"column:jsonData" json:"jsonData"` // JSON 数据字段 - UploadTime time.Time `gorm:"column:uploadTime" json:"uploadTime"` - UploadedCluster []Cluster `gorm:"foreignKey:dataID;references:id" json:"Clusters"` // 关联 Cluster 数据 -} - -type DataScheduleReq struct { - PackageID cdssdk.PackageID `json:"packageID"` - DataType string `json:"dataType"` - Clusters []Cluster `json:"clusters"` -} - -type codeRepository struct { - RepositoryName string - ClusterID ClusterID -} - -type DataScheduleResp struct { - Results []sch.DataScheduleResult `json:"results"` -} - -func (c *Client) DataSchedule(req DataScheduleReq) (*DataScheduleResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/jobSet/schedule") - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - contType := resp.Header.Get("Content-Type") - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[DataScheduleResp] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return nil, fmt.Errorf("parsing response: %w", err) - } - - if codeResp.Code == ResponseCodeOK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() - } - - return nil, fmt.Errorf("unknow response content type: %s", contType) -} - -type UploadReq struct { - Type string `json:"type"` - Source UploadSource `json:"source"` - Target UploadTarget `json:"target"` - StorageIDs []cdssdk.StorageID `json:"storageIDs"` -} - -type UploadSource interface { - Noop() -} - -var UploadSourceTypeUnion = types.NewTypeUnion[UploadSource]( - (*PackageSource)(nil), - (*UrlSource)(nil), -) - -var _ = serder.UseTypeUnionInternallyTagged(&UploadSourceTypeUnion, "type") - -type PackageSource struct { - serder.Metadata `union:"packageSource"` - UploadSourceBase - Type string `json:"type"` - PackageID cdssdk.PackageID `json:"packageID"` -} - -type UrlSource struct { - serder.Metadata `union:"urlSource"` - UploadSourceBase - Type string `json:"type"` - Url string `json:"url"` -} - -type UploadSourceBase struct{} - -func (d *UploadSourceBase) Noop() {} - -type UploadTarget interface { - Noop() -} - -var UploadTargetTypeUnion = types.NewTypeUnion[UploadTarget]( - (*UrlTarget)(nil), - (*ApiTarget)(nil), -) - -var _ = serder.UseTypeUnionInternallyTagged(&UploadTargetTypeUnion, "type") - -type UrlTarget struct { - serder.Metadata `union:"url"` - UploadTargetBase - Clusters []ClusterID `json:"clusters"` -} - -type ApiTarget struct { - serder.Metadata `union:"api"` - UploadTargetBase - Clusters []ClusterID `json:"clusters"` -} - -type UploadTargetBase struct{} - -func (d *UploadTargetBase) Noop() {} - -type UploadResp struct { - PackageID cdssdk.PackageID `json:"packageID"` - JsonData string `json:"jsonData"` -} - -func (c *Client) Upload(req UploadReq) (*UploadResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/data/upload") - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - contType := resp.Header.Get("Content-Type") - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[UploadResp] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return nil, fmt.Errorf("parsing response: %w", err) - } - - if codeResp.Code == ResponseCodeOK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() - } - - return nil, fmt.Errorf("unknow response content type: %s", contType) -} diff --git a/sdks/pcmscheduler/access.go b/sdks/pcmscheduler/access.go new file mode 100644 index 0000000..fa0c296 --- /dev/null +++ b/sdks/pcmscheduler/access.go @@ -0,0 +1,41 @@ +package sch + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) + +type AccessRequest struct { + ID uint `gorm:"column:id" json:"ID"` + BindingID int64 `gorm:"column:binding_id" json:"packageId"` + ApplicantID int64 `gorm:"column:applicant_id" json:"applicantID"` + Reason string `gorm:"column:reason" json:"reason"` + ExpirationDate time.Time `gorm:"column:expiration_date" json:"expirationDate"` + Status string `gorm:"column:status" json:"status"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` +} + +type PermissionApply struct { + BindingID int64 `gorm:"column:binding_id" json:"bindingID"` + ApplicantID cdssdk.UserID `gorm:"column:applicant_id" json:"applicantID"` // 申请人ID + OwnerID cdssdk.UserID `gorm:"column:data_owner_id" json:"ownerID"` // 数据拥有者ID + Reason string `gorm:"column:apply_reason" json:"reason"` // 申请理由 + ExpirationDate time.Time `gorm:"column:expiration_date" json:"expirationDate"` // 过期时间 + Status string `gorm:"column:status" json:"status"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` +} + +type PermissionApproval struct { + ID int64 `json:"id"` + Reason string `gorm:"column:reject_reason" json:"reason"` + Status string `gorm:"column:status" json:"status"` +} + +type AccessLog struct { + //ID uint `json:"id"` + BindingID int64 `gorm:"column:binding_id" json:"packageId"` + ApplicantID cdssdk.UserID `gorm:"column:applicant_id" json:"applicantID"` + OwnerID cdssdk.UserID `gorm:"column:data_owner_id" json:"OwnerID"` + AccessTime time.Time `gorm:"column:access_time" json:"accessTime"` + AccessType string `gorm:"column:access_type" json:"accessType"` +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index e5b69b0..f06eca1 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -14,10 +14,10 @@ type GetClusterInfoReq struct { IDs []schsdk.ClusterID `json:"clusterIDs"` } -type GetClusterInfoResp struct { - Data []ClusterDetail `json:"data"` - TraceId string `json:"traceId"` -} +//type GetClusterInfoResp struct { +// Data []ClusterDetail `json:"data"` +// TraceId string `json:"traceId"` +//} func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) { targetUrl, err := url.JoinPath(c.baseURL, "/queryResources") @@ -49,7 +49,7 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) type CreateJobReq struct { JobResources schsdk.JobResources `json:"jobResources"` - DataDistribute DataDistribute `json:"dataDistribute"` + DataDistribute DataDistribute `json:"dataDistributes"` } type DataDistribute struct { @@ -61,29 +61,30 @@ type DataDistribute struct { type DataDetail struct { ClusterID schsdk.ClusterID `json:"clusterID"` + StorageID cdssdk.StorageID `json:"storageID"` JsonData string `json:"jsonData"` } type DatasetDistribute struct { - //DataName string `json:"dataName"` + DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } type CodeDistribute struct { - //DataName string `json:"dataName"` + DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } type ImageDistribute struct { - //DataName string `json:"dataName"` + DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } type ModelDistribute struct { - //DataName string `json:"dataName"` + DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } @@ -101,11 +102,18 @@ type ScheduleData struct { } func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/jobSet/submit") + targetUrl, err := url.JoinPath(c.baseURL, "/createTask") if err != nil { return nil, err } + // 将req转换成json，并打印 + req2, err := serder.ObjectToJSONEx(req) + if err != nil { + return nil, fmt.Errorf("request to json: %w", err) + } + fmt.Println(string(req2)) + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, }) @@ -137,11 +145,10 @@ type RunJobReq struct { } type DataScheduleResult struct { - Clusters DataDetail `json:"clusters"` - PackageID cdssdk.PackageID `json:"packageID"` - PackageFullPath string `json:"packageFullPath"` - Status bool `json:"status"` - Msg string `json:"msg"` + Clusters DataDetail `json:"clusters"` + PackageID cdssdk.PackageID `json:"packageID"` + Status bool `json:"status"` + Msg string `json:"msg"` } type DataScheduleResults struct { @@ -150,7 +157,7 @@ type DataScheduleResults struct { } func (c *Client) RunJob(req RunJobReq) error { - targetUrl, err := url.JoinPath(c.baseURL, "/jobSet/submit") + targetUrl, err := url.JoinPath(c.baseURL, "runTask") if err != nil { return err } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 3f7dc3e..fefa7c8 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -25,6 +25,13 @@ const ( OrderByName = "name" OrderBySize = "size" OrderByTime = "time" + + StorageTypeURL = "url" + StorageTypeJCS = "jcs" + + RejectedStatus = "rejected" + PendingStatus = "pending" + ApprovedStatus = "approved" ) type TaskID int64 @@ -38,8 +45,13 @@ type ClusterDetail struct { Region string `json:"region"` // 资源类型 Resources2 []ResourceData `json:"resources1,omitempty"` - //Resources []ResourceData `json:"resources"` - Resources []TmpResourceData `json:"resources"` + //Resources2 []ResourceData `json:"resources"` + Resources []ClusterResource `json:"resources"` +} + +type ClusterResource struct { + Resource TmpResourceData `json:"resource"` + BaseResources []TmpResourceData `json:"baseResources"` } type TmpResourceData struct { @@ -63,6 +75,8 @@ var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( (*GPGPUResourceData)(nil), (*StorageResourceData)(nil), (*MemoryResourceData)(nil), + (*BalanceResourceData)(nil), + (*RateResourceData)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "type") @@ -156,6 +170,39 @@ type MemoryResourceData struct { Available UnitValue[float64] `json:"available"` } +type BalanceResourceData struct { + serder.Metadata `union:"BALANCE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type RateResourceData struct { + serder.Metadata `union:"RATE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceRange struct { + UserID cdssdk.UserID `json:"userID"` + Type ResourceType `json:"type"` + GPU Range `json:"gpu"` + GPUNumber int `json:"gpuNumber"` + CPU Range `json:"cpu"` + Memory Range `json:"memory"` + Storage Range `json:"storage"` +} + +type Range struct { + Min float64 `json:"min"` + Max float64 `json:"max"` +} + type ResourcePriority interface { Noop() } @@ -195,9 +242,9 @@ type BiasPriority struct { } type UploadParams struct { - DataType string `json:"dataType"` - UploadInfo UploadInfo `json:"uploadInfo"` - UploadPriority UploadPriority `json:"uploadPriority"` + DataType string `json:"dataType"` + UploadInfo UploadInfo `json:"uploadInfo"` + //UploadPriority UploadPriority `json:"uploadPriority"` } type UploadInfo interface { @@ -214,8 +261,9 @@ var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type") type LocalUploadInfo struct { serder.Metadata `union:"local"` UploadInfoBase - Type string `json:"type"` - LocalPath string `json:"localPath"` + Type string `json:"type"` + LocalPath string `json:"localPath"` + ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` } type RemoteUploadInfo struct { @@ -261,12 +309,125 @@ type UploadPriorityBase struct{} func (d *UploadPriorityBase) Noop() {} type QueryData struct { - DataType string `json:"dataType" binding:"required"` - UserID cdssdk.UserID `json:"userID" binding:"required"` - //ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + DataType string `json:"dataType" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` Path string `json:"path"` PackageID cdssdk.PackageID `json:"packageID" binding:"required"` CurrentPage int `json:"currentPage" binding:"required"` PageSize int `json:"pageSize" binding:"required"` OrderBy string `json:"orderBy" binding:"required"` } + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*DatasetBinding)(nil), + (*ModelBinding)(nil), + (*CodeBinding)(nil), + (*ImageBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct{} + +func (d *DataBindingBase) Noop() {} + +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageIDs []cdssdk.PackageID `json:"packageIDs"` +} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + Env string `json:"env"` + Version string `json:"version"` + PackageIDs []cdssdk.PackageID `json:"packageIDs"` +} + +type CodeBinding struct { + serder.Metadata `union:"code"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + ImageID string `json:"imageID"` + ObjectID cdssdk.ObjectID `json:"objectID"` + PackageID cdssdk.PackageID `json:"packageID"` +} + +type ImageBinding struct { + serder.Metadata `union:"image"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Architecture string `json:"architecture"` + ResourceType string `json:"resourceType"` + Tags []string `json:"tags"` + PackageIDs []cdssdk.PackageID `json:"packageIDs"` +} + +type ClusterImage struct { + ID int64 `json:"id"` + Name string `json:"name"` + ClusterID schsdk.ClusterID `json:"clusterID"` + ClusterImageID string `json:"clusterImageID"` + CardType string `json:"cardType"` +} + +type QueryBindingDataParam interface { + Noop() +} + +var QueryBindingDataParamTypeUnion = types.NewTypeUnion[QueryBindingDataParam]( + (*PrivateLevel)(nil), + (*ApplyLevel)(nil), + (*PublicLevel)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&QueryBindingDataParamTypeUnion, "type") + +type QueryBindingDataParamBase struct{} + +func (d *QueryBindingDataParamBase) Noop() {} + +type PrivateLevel struct { + serder.Metadata `union:"private"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + BindingID int64 `json:"bindingID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type ApplyLevel struct { + serder.Metadata `union:"apply"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type PublicLevel struct { + serder.Metadata `union:"public"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index e37db88..10bd51b 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -96,17 +96,23 @@ type JobResources struct { } type ClusterInfo struct { - ClusterID ClusterID `json:"clusterID"` - Resources []Resource `json:"resources"` - Runtime JobRuntimeInfo `json:"runtime"` + ClusterID ClusterID `json:"clusterID"` + Resources []JobResource `json:"resources"` + Runtime PCMJobRuntimeInfo `json:"runtime"` } -type Resource struct { - Resource []JobResource `json:"resource"` +type PCMJobRuntimeInfo struct { + Command string `json:"command"` + Envs map[string]interface{} `json:"envs"` + Params map[string]interface{} `json:"params"` } +//type Resource struct { +// Resource []JobResource `json:"resource"` +//} + type JobResource interface { - Noop() + Noop2() } var JobResourceTypeUnion = types.NewTypeUnion[JobResource]( @@ -123,7 +129,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type") type JobResourceBase struct{} -func (d *JobResourceBase) Noop() {} +func (d *JobResourceBase) Noop2() {} type CPU struct { serder.Metadata `union:"CPU"` diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index bc4f74d..33e51e2 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -1,6 +1,7 @@ package uploadersdk import ( + sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "time" ) @@ -8,8 +9,9 @@ import ( type ClusterID string type BlockChain struct { - ObjectID cdssdk.ObjectID `gorm:"column:object_id" json:"objectID"` - BlockChainID string `gorm:"column:blockChain_id" json:"blockChainID"` + ObjectID cdssdk.ObjectID `gorm:"column:object_id" json:"objectID"` + BlockChainID string `gorm:"column:blockchain_id" json:"blockChainID"` + BlockChainType string `gorm:"column:blockchain_type" json:"blockChainType"` //FileName string `gorm:"column:file_name" json:"fileName"` //FileHash string `gorm:"column:file_hash" json:"fileHash"` //FileSize int64 `gorm:"column:file_size" json:"fileSize"` @@ -19,14 +21,33 @@ func (BlockChain) TableName() string { return "BlockChain" // 确保和数据库中的表名一致 } -type BindingData struct { - ID DataID `gorm:"column:ID" json:"ID"` - UserID cdssdk.UserID `gorm:"column:userID" json:"userID"` - BindingName string `gorm:"column:bindingName" json:"bindingName"` - BindingType string `gorm:"column:bindingType" json:"bindingType"` +type Binding struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` } -func (BindingData) TableName() string { +type BindingAccessData struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + ApplicantID cdssdk.UserID `json:"applicant_id"` + Status string `json:"status"` +} + +type BindingDetail struct { + ID DataID `json:"ID"` + Name string `gorm:"column:name" json:"Name"` + Info sch.DataBinding `json:"info"` + Packages []Package `json:"packages"` + Status string `json:"status"` +} + +func (Binding) TableName() string { return "BindingData" // 确保和数据库中的表名一致 } diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go index 254240d..f9951c2 100644 --- a/sdks/uploader/uploader.go +++ b/sdks/uploader/uploader.go @@ -6,17 +6,19 @@ import ( sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" "net/url" "strings" + "time" ) type DataID int64 type FolderID int64 type Cluster struct { - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"dataID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"PackageID"` ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` StorageID cdssdk.StorageID `gorm:"column:storage_id" json:"storageID"` } @@ -29,12 +31,14 @@ type Package struct { UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` DataType string `gorm:"column:data_type" json:"dataType"` JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` - //UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 + Versions []PackageVersion `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 } @@ -42,29 +46,75 @@ type PackageDAO struct { UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` DataType string `gorm:"column:data_type" json:"dataType"` JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 + Versions []PackageVersion `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` +} + +type PackageVersion struct { + ParentPackageID cdssdk.PackageID `gorm:"column:parent_package_id" json:"parentPackageID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + Version int64 `gorm:"column:package_version" json:"version"` +} + +func (PackageVersion) TableName() string { + return "packageVersion" // 确保和数据库中的表名一致 } type DataScheduleReq struct { - PackageID cdssdk.PackageID `json:"packageID"` - DataType string `json:"dataType"` - Clusters []Cluster `json:"clusters"` + PackageID cdssdk.PackageID `json:"packageID"` + DataType string `json:"dataType"` + ScheduleTarget ScheduleTarget `json:"scheduleTarget"` } -type codeRepository struct { - RepositoryName string - ClusterID ClusterID +type ScheduleTarget interface { + Noop() } -type DataScheduleResp struct { - Results []sch.DataScheduleResult `json:"results"` +var DataScheduleTargetTypeUnion = types.NewTypeUnion[ScheduleTarget]( + (*JCSScheduleTarget)(nil), + (*UrlScheduleTarget)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataScheduleTargetTypeUnion, "type") + +type ScheduleTargetBase struct{} + +func (d *ScheduleTargetBase) Noop() {} + +type JCSScheduleTarget struct { + ScheduleTargetBase + UserID cdssdk.UserID `json:"userID"` + ScheduleStorages []ScheduleStorage `json:"scheduleStorages"` +} + +type UrlScheduleTarget struct { + ScheduleTargetBase + ScheduleUrls []ScheduleUrl `json:"scheduleUrls"` } -func (c *Client) DataSchedule(req DataScheduleReq) (*DataScheduleResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/jobSet/schedule") +type ScheduleUrl struct { + ClusterID ClusterID `json:"clusterID"` + //RepositoryName string `json:"repositoryName"` + JsonData string `json:"jsonData"` +} + +type ScheduleStorage struct { + StorageID cdssdk.StorageID `json:"storageID"` + RootPath string `json:"rootPath"` +} + +//type DataScheduleResp struct { +// Results []sch.DataScheduleResult `json:"data"` +//} + +func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule") if err != nil { return nil, err } @@ -78,13 +128,14 @@ func (c *Client) DataSchedule(req DataScheduleReq) (*DataScheduleResp, error) { contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[DataScheduleResp] + var codeResp response[[]sch.DataScheduleResult] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { return nil, fmt.Errorf("parsing response: %w", err) } if codeResp.Code == ResponseCodeOK { - return &codeResp.Data, nil + + return codeResp.Data, nil } return nil, codeResp.ToError() @@ -94,10 +145,10 @@ func (c *Client) DataSchedule(req DataScheduleReq) (*DataScheduleResp, error) { } type UploadReq struct { - Type string `json:"type"` - Source UploadSource `json:"source"` - Target UploadTarget `json:"target"` - StorageIDs []cdssdk.StorageID `json:"storageIDs"` + DataType string `json:"dataType"` + Source UploadSource `json:"source"` + Target UploadTarget `json:"target"` + //StorageIDs []cdssdk.StorageID `json:"storageIDs"` } type UploadSource interface { @@ -143,7 +194,8 @@ var _ = serder.UseTypeUnionInternallyTagged(&UploadTargetTypeUnion, "type") type UrlTarget struct { serder.Metadata `union:"url"` UploadTargetBase - Clusters []ClusterID `json:"clusters"` + Clusters []ClusterID `json:"clusters"` + JCSUploadInfo cdsapi.ObjectUploadInfo `form:"JCSUploadInfo"` } type ApiTarget struct { @@ -163,7 +215,7 @@ type UploadResp struct { } func (c *Client) Upload(req UploadReq) (*UploadResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/data/upload") + targetUrl, err := url.JoinPath(c.baseURL, "/dataUpload") if err != nil { return nil, err } From aaf21a2f9167f9239b0c4342c0cca3727d508a25 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 17 Jan 2025 16:36:30 +0800 Subject: [PATCH 019/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=E6=8F=90=E7=A4=BAjson=E5=BA=93=E7=9A=84BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/serder/serder_test.go | 16 ++++++++++++++++ utils/serder/union_handler.go | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/utils/serder/serder_test.go b/utils/serder/serder_test.go index 4914531..bdb3101 100644 --- a/utils/serder/serder_test.go +++ b/utils/serder/serder_test.go @@ -648,4 +648,20 @@ func Test_ObjectToJSON2(t *testing.T) { So(err, ShouldBeNil) So(ret, ShouldResemble, val) }) + +} + +func Test_ObjectToJSON3(t *testing.T) { + Convey("反序列化TypeUnion时，JSON中对应字段类型不对", t, func() { + type Base interface{} + + union := types.NewTypeUnion[Base](&St1{}, &St2{}) + UseTypeUnionInternallyTagged(&union, "Type") + + v, err := JSONToObjectEx[[]Base]([]byte("[{\"Type\":\"St2\", \"Val\":[]}]")) + t.Logf("err: %v", err) + t.Logf("v: %+v", v[0]) + So(err, ShouldNotBeNil) + // So(ret, ShouldResemble, val) + }) } diff --git a/utils/serder/union_handler.go b/utils/serder/union_handler.go index ce4ac3e..cd98a02 100644 --- a/utils/serder/union_handler.go +++ b/utils/serder/union_handler.go @@ -254,7 +254,7 @@ func (e *InternallyTaggedDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iter // 否则New出来的是会是**T，这将导致后续的反序列化出问题 if typ.Kind() == reflect.Pointer { val := reflect.New(typ.Elem()) - raw.ToVal(val.Interface()) + raw.ToVal(val.Interface()) // TODO 使用的库丢失了ToVal期间的错误信息，考虑换个库 retVal := reflect.NewAt(e.union.Union.UnionType, ptr) retVal.Elem().Set(val) From 3da3346411ce383051e1d82b64f77c5cc25ca520 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 20 Jan 2025 10:55:46 +0800 Subject: [PATCH 020/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0ECMultiplier=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/storage.go | 38 +++++++++++++++++++++++++++++++++ sdks/storage/storage_feature.go | 16 ++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/sdks/storage/storage.go b/sdks/storage/storage.go index 5986f2f..f8e686b 100644 --- a/sdks/storage/storage.go +++ b/sdks/storage/storage.go @@ -39,12 +39,30 @@ type StorageType interface { } var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[StorageType]( + (*MashupStorageType)(nil), (*LocalStorageType)(nil), (*OBSType)(nil), (*OSSType)(nil), (*COSType)(nil), + (*EFileType)(nil), )), "type") +// 多种存储服务的混合存储服务。需谨慎选择存储服务的组合，避免出Bug +type MashupStorageType struct { + serder.Metadata `union:"Mashup"` + Type string `json:"type"` + Agent StorageType `json:"agent"` // 创建Agent时，使用的存储服务类型 + Feature StorageType `json:"feature"` // 根据Feature创建组件时使用的存储服务类型 +} + +func (a *MashupStorageType) GetStorageType() string { + return "Mashup" +} + +func (a *MashupStorageType) String() string { + return "Mashup" +} + type LocalStorageType struct { serder.Metadata `union:"Local"` Type string `json:"type"` @@ -112,3 +130,23 @@ func (a *COSType) GetStorageType() string { func (a *COSType) String() string { return "COS" } + +type EFileType struct { + serder.Metadata `union:"EFile"` + Type string `json:"type"` + TokenURL string `json:"tokenURL"` + APIURL string `json:"apiURL"` + TokenExpire int `json:"tokenExpire"` // 单位秒 + User string `json:"user"` + Password string `json:"password"` + OrgID string `json:"orgID"` + ClusterID string `json:"clusterID"` +} + +func (a *EFileType) GetStorageType() string { + return "EFile" +} + +func (a *EFileType) String() string { + return "EFile" +} diff --git a/sdks/storage/storage_feature.go b/sdks/storage/storage_feature.go index db6f9f2..3e97cda 100644 --- a/sdks/storage/storage_feature.go +++ b/sdks/storage/storage_feature.go @@ -18,6 +18,7 @@ var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Storage (*MultipartUploadFeature)(nil), (*InternalServerlessCallFeature)(nil), (*S2STransferFeature)(nil), + (*ECMultiplierFeature)(nil), )), "type") type TempStore struct { @@ -94,3 +95,18 @@ func (f *S2STransferFeature) GetFeatureType() string { func (f *S2STransferFeature) String() string { return "S2STransfer" } + +// 存储服务提供了能进行EC计算的接口 +type ECMultiplierFeature struct { + serder.Metadata `union:"ECMultiplier"` + Type string `json:"type"` + TempDir string `json:"tempDir"` // 临时文件存放目录 +} + +func (f *ECMultiplierFeature) GetFeatureType() string { + return "ECMultiplier" +} + +func (f *ECMultiplierFeature) String() string { + return "ECMultiplier" +} From e747959082dec1f75b12f7bccbff39245a6771a3 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 20 Jan 2025 11:06:08 +0800 Subject: [PATCH 021/155] =?UTF-8?q?SharedStore=E9=87=8D=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E4=B8=BAPublicStore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/shared_storage.go | 14 +++++++------- sdks/storage/storage.go | 3 +-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/sdks/storage/shared_storage.go b/sdks/storage/shared_storage.go index 7364143..59d1341 100644 --- a/sdks/storage/shared_storage.go +++ b/sdks/storage/shared_storage.go @@ -7,27 +7,27 @@ import ( "gitlink.org.cn/cloudream/common/utils/serder" ) -type SharedStoreConfig interface { - GetSharedStoreType() string +type PublicStoreConfig interface { + GetPublicStoreType() string // 输出调试用的字符串，不要包含敏感信息 String() string } -var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[SharedStoreConfig]( - (*LocalSharedStorage)(nil), +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[PublicStoreConfig]( + (*LocalPublicStorage)(nil), )), "type") -type LocalSharedStorage struct { +type LocalPublicStorage struct { serder.Metadata `union:"Local"` Type string `json:"type"` // 调度Package时的Package的根路径 LoadBase string `json:"loadBase"` } -func (s *LocalSharedStorage) GetSharedStoreType() string { +func (s *LocalPublicStorage) GetPublicStoreType() string { return "Local" } -func (s *LocalSharedStorage) String() string { +func (s *LocalPublicStorage) String() string { return fmt.Sprintf("Local[LoadBase=%v]", s.LoadBase) } diff --git a/sdks/storage/storage.go b/sdks/storage/storage.go index f8e686b..cfbca6d 100644 --- a/sdks/storage/storage.go +++ b/sdks/storage/storage.go @@ -17,8 +17,7 @@ type Storage struct { // 分片存储服务的配置数据 ShardStore ShardStoreConfig `json:"shardStore" gorm:"column:ShardStore; type:json; serializer:union"` // 共享存储服务的配置数据 - SharedStore SharedStoreConfig `json:"sharedStore" gorm:"column:SharedStore; type:json; serializer:union"` - // SharedStore + PublicStore PublicStoreConfig `json:"publicStore" gorm:"column:PublicStore; type:json; serializer:union"` // 存储服务拥有的特别功能 Features []StorageFeature `json:"features" gorm:"column:Features; type:json; serializer:union"` } From eada8937a4a2823d30d4a76cca381f6a271c0601 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 20 Jan 2025 11:10:36 +0800 Subject: [PATCH 022/155] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/{shared_storage.go => public_storage.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sdks/storage/{shared_storage.go => public_storage.go} (100%) diff --git a/sdks/storage/shared_storage.go b/sdks/storage/public_storage.go similarity index 100% rename from sdks/storage/shared_storage.go rename to sdks/storage/public_storage.go From 7890ee37b75e9a4068a228d5ff81bc6d6ef26a9c Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 23 Jan 2025 15:12:16 +0800 Subject: [PATCH 023/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/ioswitch/dag/node.go | 4 ++++ sdks/storage/filehash.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/pkgs/ioswitch/dag/node.go b/pkgs/ioswitch/dag/node.go index e8f0f1d..6fbe735 100644 --- a/pkgs/ioswitch/dag/node.go +++ b/pkgs/ioswitch/dag/node.go @@ -457,6 +457,10 @@ func (s StreamOutputSlot) Var() *StreamVar { return s.Node.OutputStreams().Get(s.Index) } +func (s StreamOutputSlot) ToSlot(slot StreamInputSlot) { + s.Var().To(slot.Node, slot.Index) +} + type StreamInputSlot struct { Node Node Index int diff --git a/sdks/storage/filehash.go b/sdks/storage/filehash.go index 9fecbaf..d4f9df5 100644 --- a/sdks/storage/filehash.go +++ b/sdks/storage/filehash.go @@ -70,6 +70,10 @@ func NewFullHash(hash []byte) FileHash { return FileHash(FullHashPrefix + strings.ToUpper(hex.EncodeToString(hash))) } +func NewFullHashFromString(hashStr string) FileHash { + return FileHash(FullHashPrefix + strings.ToUpper(hashStr)) +} + func CalculateCompositeHash(segmentHashes [][]byte) FileHash { data := make([]byte, len(segmentHashes)*32) for i, segmentHash := range segmentHashes { From e6c2be9910aff0b984ae8b8896f53c62fd02049e Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 24 Jan 2025 09:16:32 +0800 Subject: [PATCH 024/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/serder/serder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/serder/serder.go b/utils/serder/serder.go index 268215d..d3cfcd4 100644 --- a/utils/serder/serder.go +++ b/utils/serder/serder.go @@ -77,6 +77,8 @@ func JSONToObjectStreamEx[T any](stream io.Reader) (T, error) { } // 将对象转为JSON字符串。如果需要支持解析TypeUnion类型，则使用"Ex"结尾的同名函数。 +// +// 注：[]byte会被base64编码，如果要JSON内容要给外部解析，那么应该避免使用[]byte。 func ObjectToJSON(obj any) ([]byte, error) { return json.Marshal(obj) } From d60ca055b740794a28fdcef97d1ee0bf20d6a2ed Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Sat, 25 Jan 2025 08:00:32 +0800 Subject: [PATCH 025/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=8F=90=E4=BA=A4=E7=AD=89=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 14 ++-- sdks/pcmscheduler/client.go | 6 ++ sdks/pcmscheduler/jobset.go | 21 +++-- sdks/pcmscheduler/models.go | 24 ++++-- sdks/scheduler/models.go | 5 +- sdks/uploader/models.go | 151 +++++++++++++++++++++++++++++++--- sdks/uploader/uploader.go | 121 ++++++--------------------- utils/config/config.go | 2 + 8 files changed, 217 insertions(+), 127 deletions(-) diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go index b85b859..a7cb793 100644 --- a/sdks/blockchain/blockchain.go +++ b/sdks/blockchain/blockchain.go @@ -16,35 +16,39 @@ type InvokeReq struct { Args []string `json:"args"` } -func (c *Client) BlockChainInvoke(req InvokeReq) error { +func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { targetUrl, err := url.JoinPath(c.baseURL, "/jcc-bcos/contract/invoke") if err != nil { return err } - token, err := c.getToken() - if err != nil { - return err - } + //token, err := c.getToken() + //if err != nil { + // return err + //} header := make(map[string]string) header["Content-Type"] = http2.ContentTypeJSON header["Authorization"] = token + println("token: " + token) resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, Header: header, }) if err != nil { + println(err) return err } var codeResp response[string] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + println(err) return fmt.Errorf("parsing response: %w", err) } if codeResp.Code != ResponseCodeOK { + println(codeResp.ToError().Message) return codeResp.ToError() } diff --git a/sdks/pcmscheduler/client.go b/sdks/pcmscheduler/client.go index a058dc7..fde07fc 100644 --- a/sdks/pcmscheduler/client.go +++ b/sdks/pcmscheduler/client.go @@ -12,6 +12,12 @@ type response[T any] struct { Data T `json:"data"` } +type respons2[T any] struct { + Code int `json:"code"` + Message string `json:"msg"` + Data T `json:"data"` +} + const ( ResponseCodeOK int = 200 ) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index f06eca1..0248ff0 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -48,6 +48,8 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) } type CreateJobReq struct { + //Name string `json:"name"` + //Description string `json:"description"` JobResources schsdk.JobResources `json:"jobResources"` DataDistribute DataDistribute `json:"dataDistributes"` } @@ -123,7 +125,7 @@ func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[CreateJobResp] + var codeResp respons2[CreateJobResp] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { return nil, fmt.Errorf("parsing response: %w", err) } @@ -132,7 +134,9 @@ func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { return &codeResp.Data, nil } - return nil, codeResp.ToError() + //return nil, codeResp.ToError() + println(codeResp.Message) + return nil, err } return nil, fmt.Errorf("unknow response content type: %s", contType) @@ -145,10 +149,11 @@ type RunJobReq struct { } type DataScheduleResult struct { - Clusters DataDetail `json:"clusters"` - PackageID cdssdk.PackageID `json:"packageID"` - Status bool `json:"status"` - Msg string `json:"msg"` + Clusters []DataDetail `json:"clusters"` + PackageID cdssdk.PackageID `json:"packageID"` + PackageFullPath string `json:"packageFullPath"` + Status bool `json:"status"` + Msg string `json:"msg"` } type DataScheduleResults struct { @@ -171,7 +176,7 @@ func (c *Client) RunJob(req RunJobReq) error { contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[string] + var codeResp respons2[string] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { return fmt.Errorf("parsing response: %w", err) } @@ -180,7 +185,7 @@ func (c *Client) RunJob(req RunJobReq) error { return nil } - return codeResp.ToError() + return fmt.Errorf("error: %s", codeResp.Message) } return fmt.Errorf("unknow response content type: %s", contType) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index fefa7c8..0a09be7 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -32,6 +32,16 @@ const ( RejectedStatus = "rejected" PendingStatus = "pending" ApprovedStatus = "approved" + RevokedStatus = "revoked" + CancelStatus = "cancel" + ExpiredStatus = "expired" + + ApplyAccess = "apply" + PrivateAccess = "private" + PublicAccess = "public" + + PreferencePriority = "preference" + SpecifyClusterPriority = "specify" ) type TaskID int64 @@ -269,10 +279,11 @@ type LocalUploadInfo struct { type RemoteUploadInfo struct { serder.Metadata `union:"url"` UploadInfoBase - Type string `json:"type"` - Url string `json:"url"` - DataName string `json:"dataName"` - TargetClusters []schsdk.ClusterID `json:"targetClusters"` + Type string `json:"type"` + Url string `json:"url"` + DataName string `json:"dataName"` + Cluster schsdk.ClusterID `json:"clusterID"` + PackageID cdssdk.PackageID `json:"packageID"` } type UploadInfoBase struct{} @@ -428,6 +439,7 @@ type ApplyLevel struct { type PublicLevel struct { serder.Metadata `union:"public"` QueryBindingDataParamBase - Type string `json:"type" binding:"required"` - Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 + UserID cdssdk.UserID `json:"userID" binding:"required"` + Type string `json:"type" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 10bd51b..906796d 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -86,12 +86,15 @@ type PCMJobInfo struct { serder.Metadata `union:"PCM"` JobInfoBase Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` Files JobFilesInfo `json:"files"` JobResources JobResources `json:"jobResources"` } type JobResources struct { - ScheduleStrategy string `json:"scheduleStrategy"` //任务分配策略：负载均衡、积分优先、随机分配等 + //任务分配策略：负载均衡、积分优先、随机分配等，dataLocality, leastLoadFirst + ScheduleStrategy string `json:"scheduleStrategy"` Clusters []ClusterInfo `json:"clusters"` } diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 33e51e2..82c7ee1 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -1,8 +1,11 @@ package uploadersdk import ( + "gitlink.org.cn/cloudream/common/pkgs/types" sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/serder" "time" ) @@ -22,11 +25,13 @@ func (BlockChain) TableName() string { } type Binding struct { - ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - Name string `gorm:"column:name" json:"Name"` - DataType string `gorm:"column:data_type" json:"dataType"` - Content string `gorm:"column:content" json:"Content"` + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } type BindingAccessData struct { @@ -35,16 +40,19 @@ type BindingAccessData struct { Name string `gorm:"column:name" json:"Name"` DataType string `gorm:"column:data_type" json:"dataType"` Content string `gorm:"column:content" json:"Content"` - ApplicantID cdssdk.UserID `json:"applicant_id"` - Status string `json:"status"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + ApplicantID cdssdk.UserID `gorm:"column:applicant_id" json:"applicantID"` + Status string `gorm:"column:status" json:"status"` } type BindingDetail struct { - ID DataID `json:"ID"` - Name string `gorm:"column:name" json:"Name"` - Info sch.DataBinding `json:"info"` - Packages []Package `json:"packages"` - Status string `json:"status"` + ID DataID `json:"ID"` + UserID cdssdk.UserID `json:"ownerID"` + Name string `json:"Name"` + Info sch.DataBinding `json:"info"` + Packages []Package `json:"packages"` + Status string `json:"status"` + AccessLevel string `json:"accessLevel"` } func (Binding) TableName() string { @@ -60,3 +68,122 @@ type Folder struct { func (Folder) TableName() string { return "folders" } + +type DataID int64 +type FolderID int64 + +type Cluster struct { + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"PackageID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + StorageID cdssdk.StorageID `gorm:"column:storage_id" json:"storageID"` +} + +func (Cluster) TableName() string { + return "uploadedCluster" // 确保和数据库中的表名一致 +} + +type Package struct { + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` + Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` + UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` + Versions []PackageVersion `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` + //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 + UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` +} + +type PackageDAO struct { + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` + UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 + Versions []PackageVersion `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` + UploadPriority string `gorm:"column:upload_priority" json:"uploadPriority"` +} + +type PackageVersion struct { + ParentPackageID cdssdk.PackageID `gorm:"column:parent_package_id" json:"parentPackageID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + Version int64 `gorm:"column:package_version" json:"version"` +} + +func (PackageVersion) TableName() string { + return "packageVersion" // 确保和数据库中的表名一致 +} + +//type PackageScheduleType interface { +// Noop() +//} +// +//var PackageScheduleTypeUnion = types.NewTypeUnion[PackageScheduleType]( +// (*PackagePreferencesSchedule)(nil), +// (*PackageSpecifyClusterSchedule)(nil), +//) +// +//var _ = serder.UseTypeUnionInternallyTagged(&PackageScheduleTypeUnion, "type") +// +//type PackageScheduleBase struct{} +// +//func (d *PackageScheduleBase) Noop() {} +// +//type PackagePreferencesSchedule struct { +// serder.Metadata `union:"region"` +// PackageScheduleBase +// Type string `json:"type"` +// a sch.Preferences +//} +// +//type PackageSpecifyClusterSchedule struct { +// serder.Metadata `union:"region"` +// PackageScheduleBase +// Type string `json:"type"` +//} + +type ScheduleTarget interface { + Noop() +} + +var DataScheduleTargetTypeUnion = types.NewTypeUnion[ScheduleTarget]( + (*JCSScheduleTarget)(nil), + (*UrlScheduleTarget)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataScheduleTargetTypeUnion, "type") + +type ScheduleTargetBase struct{} + +func (d *ScheduleTargetBase) Noop() {} + +type JCSScheduleTarget struct { + ScheduleTargetBase + UserID cdssdk.UserID `json:"userID"` + ScheduleStorages []ScheduleStorage `json:"scheduleStorages"` +} + +type UrlScheduleTarget struct { + ScheduleTargetBase + ScheduleUrls []ScheduleUrl `json:"scheduleUrls"` +} + +type ScheduleUrl struct { + ClusterID ClusterID `json:"clusterID"` + //RepositoryName string `json:"repositoryName"` + JsonData string `json:"jsonData"` +} + +type ScheduleStorage struct { + StorageID cdssdk.StorageID `json:"storageID"` + RootPath string `json:"rootPath"` +} diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go index f9951c2..a421d7b 100644 --- a/sdks/uploader/uploader.go +++ b/sdks/uploader/uploader.go @@ -4,115 +4,31 @@ import ( "fmt" "gitlink.org.cn/cloudream/common/pkgs/types" sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" - schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" "net/url" "strings" - "time" ) -type DataID int64 -type FolderID int64 - -type Cluster struct { - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"PackageID"` - ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` - StorageID cdssdk.StorageID `gorm:"column:storage_id" json:"storageID"` -} - -func (Cluster) TableName() string { - return "uploadedCluster" // 确保和数据库中的表名一致 -} - -type Package struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:create_time" json:"createTime"` - Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` - UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` - Versions []PackageVersion `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` - //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 -} - -type PackageDAO struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:create_time" json:"createTime"` - UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 - Versions []PackageVersion `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` -} - -type PackageVersion struct { - ParentPackageID cdssdk.PackageID `gorm:"column:parent_package_id" json:"parentPackageID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - Version int64 `gorm:"column:package_version" json:"version"` -} - -func (PackageVersion) TableName() string { - return "packageVersion" // 确保和数据库中的表名一致 -} - type DataScheduleReq struct { PackageID cdssdk.PackageID `json:"packageID"` DataType string `json:"dataType"` ScheduleTarget ScheduleTarget `json:"scheduleTarget"` } -type ScheduleTarget interface { - Noop() -} - -var DataScheduleTargetTypeUnion = types.NewTypeUnion[ScheduleTarget]( - (*JCSScheduleTarget)(nil), - (*UrlScheduleTarget)(nil), -) - -var _ = serder.UseTypeUnionInternallyTagged(&DataScheduleTargetTypeUnion, "type") - -type ScheduleTargetBase struct{} - -func (d *ScheduleTargetBase) Noop() {} - -type JCSScheduleTarget struct { - ScheduleTargetBase - UserID cdssdk.UserID `json:"userID"` - ScheduleStorages []ScheduleStorage `json:"scheduleStorages"` -} - -type UrlScheduleTarget struct { - ScheduleTargetBase - ScheduleUrls []ScheduleUrl `json:"scheduleUrls"` -} - -type ScheduleUrl struct { - ClusterID ClusterID `json:"clusterID"` - //RepositoryName string `json:"repositoryName"` - JsonData string `json:"jsonData"` -} - -type ScheduleStorage struct { - StorageID cdssdk.StorageID `json:"storageID"` - RootPath string `json:"rootPath"` -} - //type DataScheduleResp struct { // Results []sch.DataScheduleResult `json:"data"` //} +type TmpDataScheduleResult struct { + Cluster sch.DataDetail `json:"cluster"` + PackageID cdssdk.PackageID `json:"packageID"` + Status bool `json:"status"` + Msg string `json:"msg"` +} + func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) { targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule") if err != nil { @@ -125,17 +41,29 @@ func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, er if err != nil { return nil, err } + println(resp.Body) contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[[]sch.DataScheduleResult] + var codeResp response[[]TmpDataScheduleResult] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { return nil, fmt.Errorf("parsing response: %w", err) } if codeResp.Code == ResponseCodeOK { - - return codeResp.Data, nil + var results []sch.DataScheduleResult + for _, tmpResult := range codeResp.Data { + result := sch.DataScheduleResult{ + PackageID: tmpResult.PackageID, + Status: tmpResult.Status, + Msg: tmpResult.Msg, + Clusters: []sch.DataDetail{ + tmpResult.Cluster, + }, + } + results = append(results, result) + } + return results, nil } return nil, codeResp.ToError() @@ -194,7 +122,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&UploadTargetTypeUnion, "type") type UrlTarget struct { serder.Metadata `union:"url"` UploadTargetBase - Clusters []ClusterID `json:"clusters"` + ClusterID ClusterID `json:"clusterId"` JCSUploadInfo cdsapi.ObjectUploadInfo `form:"JCSUploadInfo"` } @@ -211,7 +139,10 @@ func (d *UploadTargetBase) Noop() {} type UploadResp struct { PackageID cdssdk.PackageID `json:"packageID"` ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` + ClusterID ClusterID `json:"clusterID"` JsonData string `json:"jsonData"` + Status bool `json:"status"` + Message string `json:"message"` } func (c *Client) Upload(req UploadReq) (*UploadResp, error) { diff --git a/utils/config/config.go b/utils/config/config.go index d0f930b..889094b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -33,6 +33,8 @@ func DefaultLoad(modeulName string, defCfg interface{}) error { // filepath.Join用于将多个路径组合成一个路径 configFilePath := filepath.Join(filepath.Dir(execPath), "..", "confs", fmt.Sprintf("%s.config.json", modeulName)) + configFilePath = "D:\\Work\\Codes\\workspace\\workspace\\scheduler\\common\\assets\\confs\\middleware.json" + return Load(configFilePath, defCfg) } From 82dbe4f260c4364d5dc7677ef3bb0c4da39789ce Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 10 Feb 2025 16:13:52 +0800 Subject: [PATCH 026/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E7=9A=84=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/access.go | 7 +++++++ sdks/pcmscheduler/jobset.go | 14 ++++++++++++-- sdks/uploader/models.go | 4 ++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/access.go b/sdks/pcmscheduler/access.go index fa0c296..8376b48 100644 --- a/sdks/pcmscheduler/access.go +++ b/sdks/pcmscheduler/access.go @@ -39,3 +39,10 @@ type AccessLog struct { AccessTime time.Time `gorm:"column:access_time" json:"accessTime"` AccessType string `gorm:"column:access_type" json:"accessType"` } + +type User struct { + ID int64 `gorm:"column:id" json:"id"` + SsoID string `gorm:"column:sso_id" json:"ssoID"` + UserName string `gorm:"column:username" json:"userName"` + Created time.Time `gorm:"column:created_at" json:"created"` +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 0248ff0..584561f 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -25,7 +25,8 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) return nil, err } - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{Body: req}) + //resp, err := http2.PostJSON(targetUrl, http2.RequestParam{Body: req}) + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{}) if err != nil { return nil, err } @@ -37,8 +38,17 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) return nil, fmt.Errorf("parsing response: %w", err) } + // 该接口传入参数后查询效率很低，所以需要在这里特殊处理 if codeResp.Code == ResponseCodeOK { - return codeResp.Data, nil + var results []ClusterDetail + for _, cluster := range codeResp.Data { + for _, id := range req.IDs { + if cluster.ClusterId == id { + results = append(results, cluster) + } + } + } + return results, nil } return nil, codeResp.ToError() diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 82c7ee1..cb5b77f 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -37,6 +37,8 @@ type Binding struct { type BindingAccessData struct { ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + UserName string `gorm:"column:username" json:"userName"` + SSOId string `gorm:"column:sso_id" json:"ssoID"` Name string `gorm:"column:name" json:"Name"` DataType string `gorm:"column:data_type" json:"dataType"` Content string `gorm:"column:content" json:"Content"` @@ -48,6 +50,8 @@ type BindingAccessData struct { type BindingDetail struct { ID DataID `json:"ID"` UserID cdssdk.UserID `json:"ownerID"` + UserName string `json:"userName"` + SSOId string `json:"ssoID"` Name string `json:"Name"` Info sch.DataBinding `json:"info"` Packages []Package `json:"packages"` From 24340c45d00e36f11c72405f114a90d79187aa28 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 10 Feb 2025 17:27:25 +0800 Subject: [PATCH 027/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/tickevent/executor.go | 4 ++-- utils/io2/stats.go | 41 +++++++++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/pkgs/tickevent/executor.go b/pkgs/tickevent/executor.go index 0bb0a22..f960404 100644 --- a/pkgs/tickevent/executor.go +++ b/pkgs/tickevent/executor.go @@ -20,7 +20,7 @@ type EventTicker[TArgs any] struct { event TickEvent[TArgs] intervalMs int doneChan chan int - done atomic.Bool + done *atomic.Bool } type Executor[TArgs any] struct { @@ -43,7 +43,7 @@ func (e *Executor[TArgs]) Start(event TickEvent[TArgs], intervalMs int, opts ... event: event, intervalMs: intervalMs, doneChan: make(chan int), - done: atomic.Bool{}, + done: &atomic.Bool{}, } ticker.done.Store(false) diff --git a/utils/io2/stats.go b/utils/io2/stats.go index b6c7956..cdc6ec5 100644 --- a/utils/io2/stats.go +++ b/utils/io2/stats.go @@ -2,21 +2,52 @@ package io2 import "io" -type Counter struct { +type counter struct { inner io.Reader count int64 } -func (c *Counter) Read(buf []byte) (n int, err error) { +func (c *counter) Read(buf []byte) (n int, err error) { n, err = c.inner.Read(buf) c.count += int64(n) return } -func (c *Counter) Count() int64 { +func (c *counter) Count() int64 { return c.count } -func NewCounter(inner io.Reader) *Counter { - return &Counter{inner: inner, count: 0} +func Counter(inner io.Reader) *counter { + return &counter{inner: inner, count: 0} +} + +type counterCloser struct { + inner io.ReadCloser + count int64 + callback func(cnt int64, err error) +} + +func (c *counterCloser) Read(buf []byte) (n int, err error) { + n, err = c.inner.Read(buf) + c.count += int64(n) + if err != nil && c.callback != nil { + c.callback(c.count, err) + c.callback = nil + } + return +} + +func (c *counterCloser) Close() error { + // 读取方主动Close，视为正常结束 + err := c.inner.Close() + if c.callback != nil { + c.callback(c.count, nil) + } + return err +} + +// 统计一个io.ReadCloser的读取字节数，在读取结束后调用callback函数。 +// 仅在读取方主动调用Close时，callback的err参数才会为nil。 +func CounterCloser(inner io.ReadCloser, callback func(cnt int64, err error)) io.ReadCloser { + return &counterCloser{inner: inner, count: 0, callback: callback} } From a48e23362de2582275cda3448780281222487382 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Feb 2025 11:14:25 +0800 Subject: [PATCH 028/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/user.go | 74 +++++++++++++++++++++++++++++++++++++ sdks/storage/models.go | 10 +++++ utils/config/config.go | 2 - 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 sdks/storage/cdsapi/user.go diff --git a/sdks/storage/cdsapi/user.go b/sdks/storage/cdsapi/user.go new file mode 100644 index 0000000..0c67235 --- /dev/null +++ b/sdks/storage/cdsapi/user.go @@ -0,0 +1,74 @@ +package cdsapi + +import ( + "net/url" + + "gitlink.org.cn/cloudream/common/consts/errorcode" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/http2" +) + +const UserCreatePath = "/user/create" + +type UserCreate struct { + Name string `json:"name" binding:"required"` +} +type UserCreateResp struct { + User cdssdk.User `json:"user"` +} + +func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { + url, err := url.JoinPath(c.baseURL, UserCreatePath) + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(url, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + codeResp, err := ParseJSONResponse[response[UserCreateResp]](resp) + if err != nil { + return nil, err + } + + if codeResp.Code == errorcode.OK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() +} + +const UserDeletePath = "/user/delete" + +type UserDelete struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` +} + +type UserDeleteResp struct{} + +func (c *Client) UserDelete(req *UserDelete) error { + url, err := url.JoinPath(c.baseURL, UserDeletePath) + if err != nil { + return err + } + + resp, err := http2.PostJSON(url, http2.RequestParam{ + Body: req, + }) + if err != nil { + return err + } + codeResp, err := ParseJSONResponse[response[UserDeleteResp]](resp) + if err != nil { + return err + } + + if codeResp.Code == errorcode.OK { + return nil + } + + return codeResp.ToError() +} diff --git a/sdks/storage/models.go b/sdks/storage/models.go index 05df652..7fcfc4d 100644 --- a/sdks/storage/models.go +++ b/sdks/storage/models.go @@ -229,6 +229,16 @@ func (b *SegmentRedundancy) CalcSegmentRange(start int64, end *int64) (segIdxSta return } +type User struct { + UserID UserID `gorm:"column:UserID; primaryKey; type:bigint" json:"userID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + Password string `gorm:"column:Password; type:varchar(255); not null" json:"password"` +} + +func (User) TableName() string { + return "User" +} + const ( PackageStateNormal = "Normal" PackageStateDeleted = "Deleted" diff --git a/utils/config/config.go b/utils/config/config.go index 889094b..d0f930b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -33,8 +33,6 @@ func DefaultLoad(modeulName string, defCfg interface{}) error { // filepath.Join用于将多个路径组合成一个路径 configFilePath := filepath.Join(filepath.Dir(execPath), "..", "confs", fmt.Sprintf("%s.config.json", modeulName)) - configFilePath = "D:\\Work\\Codes\\workspace\\workspace\\scheduler\\common\\assets\\confs\\middleware.json" - return Load(configFilePath, defCfg) } From d364e874b6edeab696e53e5ff6d2b8dfb5c8d1dd Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 12 Feb 2025 15:25:13 +0800 Subject: [PATCH 029/155] =?UTF-8?q?=E5=88=9B=E5=BB=BA=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=97=B6=E5=88=9B=E5=BB=BA=E6=A1=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/access.go | 6 ++++++ sdks/pcmscheduler/jobset.go | 14 +++++++------- sdks/pcmscheduler/models.go | 3 +++ sdks/uploader/models.go | 1 + utils/config/config.go | 2 ++ 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/sdks/pcmscheduler/access.go b/sdks/pcmscheduler/access.go index 8376b48..82b1dc2 100644 --- a/sdks/pcmscheduler/access.go +++ b/sdks/pcmscheduler/access.go @@ -46,3 +46,9 @@ type User struct { UserName string `gorm:"column:username" json:"userName"` Created time.Time `gorm:"column:created_at" json:"created"` } + +type Bucket struct { + ID cdssdk.BucketID `gorm:"column:id" json:"id"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + DataType string `gorm:"column:data_type" json:"dataType"` +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 584561f..1d05570 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -58,8 +58,8 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) } type CreateJobReq struct { - //Name string `json:"name"` - //Description string `json:"description"` + Name string `json:"name"` + Description string `json:"description"` JobResources schsdk.JobResources `json:"jobResources"` DataDistribute DataDistribute `json:"dataDistributes"` } @@ -120,11 +120,11 @@ func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { } // 将req转换成json，并打印 - req2, err := serder.ObjectToJSONEx(req) - if err != nil { - return nil, fmt.Errorf("request to json: %w", err) - } - fmt.Println(string(req2)) + //req2, err := serder.ObjectToJSONEx(req) + //if err != nil { + // return nil, fmt.Errorf("request to json: %w", err) + //} + //fmt.Println(string(req2)) resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 0a09be7..0a0e14b 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -17,6 +17,8 @@ const ( ResourceTypeStorage ResourceType = "STORAGE" ResourceTypeMemory ResourceType = "MEMORY" + Split = "/" + CODE = "code" DATASET = "dataset" IMAGE = "image" @@ -379,6 +381,7 @@ type CodeBinding struct { Description string `json:"description"` ImageID string `json:"imageID"` ObjectID cdssdk.ObjectID `json:"objectID"` + FilePath string `json:"filePath"` PackageID cdssdk.PackageID `json:"packageID"` } diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index cb5b77f..6babe86 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -29,6 +29,7 @@ type Binding struct { UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` Name string `gorm:"column:name" json:"Name"` DataType string `gorm:"column:data_type" json:"dataType"` + JsonData string `gorm:"column:json_data" json:"jsonData"` Content string `gorm:"column:content" json:"Content"` AccessLevel string `gorm:"column:access_level" json:"accessLevel"` CreateTime time.Time `gorm:"column:created_at" json:"createTime"` diff --git a/utils/config/config.go b/utils/config/config.go index d0f930b..889094b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -33,6 +33,8 @@ func DefaultLoad(modeulName string, defCfg interface{}) error { // filepath.Join用于将多个路径组合成一个路径 configFilePath := filepath.Join(filepath.Dir(execPath), "..", "confs", fmt.Sprintf("%s.config.json", modeulName)) + configFilePath = "D:\\Work\\Codes\\workspace\\workspace\\scheduler\\common\\assets\\confs\\middleware.json" + return Load(configFilePath, defCfg) } From 69164e1a93771a9b2e4ff3beb4283bda2ed3cb50 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 12 Feb 2025 15:55:26 +0800 Subject: [PATCH 030/155] =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=8F=B7=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/user.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/storage/cdsapi/user.go b/sdks/storage/cdsapi/user.go index 0c67235..7397f89 100644 --- a/sdks/storage/cdsapi/user.go +++ b/sdks/storage/cdsapi/user.go @@ -8,7 +8,7 @@ import ( "gitlink.org.cn/cloudream/common/utils/http2" ) -const UserCreatePath = "/user/create" +const UserCreatePath = "/v1/user/create" type UserCreate struct { Name string `json:"name" binding:"required"` @@ -41,7 +41,7 @@ func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { return nil, codeResp.ToError() } -const UserDeletePath = "/user/delete" +const UserDeletePath = "/v1/user/delete" type UserDelete struct { UserID cdssdk.UserID `json:"userID" binding:"required"` From 86221d8528cdfca083c4c64c94562bd6bae4cda0 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 12 Feb 2025 16:25:35 +0800 Subject: [PATCH 031/155] =?UTF-8?q?=E5=88=A0=E9=99=A4=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/config/config.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 889094b..d0f930b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -33,8 +33,6 @@ func DefaultLoad(modeulName string, defCfg interface{}) error { // filepath.Join用于将多个路径组合成一个路径 configFilePath := filepath.Join(filepath.Dir(execPath), "..", "confs", fmt.Sprintf("%s.config.json", modeulName)) - configFilePath = "D:\\Work\\Codes\\workspace\\workspace\\scheduler\\common\\assets\\confs\\middleware.json" - return Load(configFilePath, defCfg) } From c3c42c5d0a38fb56ec2e7ec26e273b6fe4eb2cf8 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 12 Feb 2025 17:02:23 +0800 Subject: [PATCH 032/155] =?UTF-8?q?=E8=B0=83=E6=95=B4uploader=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/uploader.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go index a421d7b..7a99b2c 100644 --- a/sdks/uploader/uploader.go +++ b/sdks/uploader/uploader.go @@ -91,14 +91,14 @@ var UploadSourceTypeUnion = types.NewTypeUnion[UploadSource]( var _ = serder.UseTypeUnionInternallyTagged(&UploadSourceTypeUnion, "type") type PackageSource struct { - serder.Metadata `union:"packageSource"` + serder.Metadata `union:"jcs"` UploadSourceBase Type string `json:"type"` PackageID cdssdk.PackageID `json:"packageID"` } type UrlSource struct { - serder.Metadata `union:"urlSource"` + serder.Metadata `union:"url"` UploadSourceBase Type string `json:"type"` Url string `json:"url"` @@ -122,6 +122,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&UploadTargetTypeUnion, "type") type UrlTarget struct { serder.Metadata `union:"url"` UploadTargetBase + Type string `json:"type"` ClusterID ClusterID `json:"clusterId"` JCSUploadInfo cdsapi.ObjectUploadInfo `form:"JCSUploadInfo"` } @@ -129,6 +130,7 @@ type UrlTarget struct { type ApiTarget struct { serder.Metadata `union:"api"` UploadTargetBase + Type string `json:"type"` Clusters []ClusterID `json:"clusters"` } From e0f6c8359657ac471e88b27fa2cf3f3055a819d4 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 13 Feb 2025 09:27:29 +0800 Subject: [PATCH 033/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=BA=8F=E5=88=97?= =?UTF-8?q?=E5=8C=96=E8=BF=87=E7=A8=8B=E7=9A=84=E5=9B=9E=E8=B0=83=E5=87=BD?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/serder/serder_test.go | 29 +++++++++++++++++++++++++++++ utils/serder/types/types.go | 4 ++++ utils/serder/union_handler.go | 11 +++++++++++ 3 files changed, 44 insertions(+) diff --git a/utils/serder/serder_test.go b/utils/serder/serder_test.go index bdb3101..1864199 100644 --- a/utils/serder/serder_test.go +++ b/utils/serder/serder_test.go @@ -665,3 +665,32 @@ func Test_ObjectToJSON3(t *testing.T) { // So(ret, ShouldResemble, val) }) } + +type BaseCallback interface{} +type StCallback struct { + Metadata `union:"StCallback"` + Type string + Value string +} + +func (st *StCallback) OnUnionSerializing() { + st.Value = "called" + st.Type = "StCallback" +} + +func Test_ObjectToJSONEx4(t *testing.T) { + Convey("序列化Callback", t, func() { + union := types.NewTypeUnion[BaseCallback](&StCallback{}) + UseTypeUnionInternallyTagged(&union, "Type") + + val := []BaseCallback{&StCallback{}} + data, err := ObjectToJSONEx(val) + So(err, ShouldBeNil) + + ret, err := JSONToObjectEx[[]BaseCallback](data) + So(err, ShouldBeNil) + + So(len(ret), ShouldEqual, 1) + So(ret[0].(*StCallback).Value, ShouldEqual, "called") + }) +} diff --git a/utils/serder/types/types.go b/utils/serder/types/types.go index 0046926..a1cd975 100644 --- a/utils/serder/types/types.go +++ b/utils/serder/types/types.go @@ -1,3 +1,7 @@ package types type Metadata struct{} + +type OnUnionSerializing interface { + OnUnionSerializing() +} diff --git a/utils/serder/union_handler.go b/utils/serder/union_handler.go index cd98a02..dd6fb16 100644 --- a/utils/serder/union_handler.go +++ b/utils/serder/union_handler.go @@ -8,6 +8,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/modern-go/reflect2" "gitlink.org.cn/cloudream/common/pkgs/types" + sertypes "gitlink.org.cn/cloudream/common/utils/serder/types" ref2 "gitlink.org.cn/cloudream/common/utils/reflect2" ) @@ -211,6 +212,12 @@ func (e *InternallyTaggedEncoder) Encode(ptr unsafe.Pointer, stream *jsoniter.St val = reflect2.IFaceToEFace(ptr) } + if val != nil { + if on, ok := val.(sertypes.OnUnionSerializing); ok { + on.OnUnionSerializing() + } + } + // 可以考虑检查一下Type字段有没有赋值，没有赋值则将其赋值为union Tag指定的值 stream.WriteVal(val) } @@ -292,6 +299,10 @@ func (e *ExternallyTaggedEncoder) Encode(ptr unsafe.Pointer, stream *jsoniter.St return } + if on, ok := val.(sertypes.OnUnionSerializing); ok { + on.OnUnionSerializing() + } + stream.WriteObjectStart() valType := ref2.TypeOfValue(val) if !e.union.Union.Include(valType) { From a272e3651130dbe95793687633113ff0dcc1a83f Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 13 Feb 2025 14:57:14 +0800 Subject: [PATCH 034/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/math2/math.go | 9 +++++++++ utils/serder/serder.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/utils/math2/math.go b/utils/math2/math.go index 7a8508b..db54476 100644 --- a/utils/math2/math.go +++ b/utils/math2/math.go @@ -72,3 +72,12 @@ func SplitN[T constraints.Integer](v T, n int) []T { return result } + +// 除法，如果除数为0，则返回默认值 +func DivOrDefault[T constraints.Integer](v, d T, def T) T { + if d == 0 { + return def + } + + return v / d +} diff --git a/utils/serder/serder.go b/utils/serder/serder.go index d3cfcd4..c665201 100644 --- a/utils/serder/serder.go +++ b/utils/serder/serder.go @@ -65,6 +65,9 @@ func JSONToObjectExRaw(data []byte, ret any) error { } // 将JSON字符串转为对象。支持TypeUnion。 +// +// 如果发现反序列化后的结果不对，但没有返回错误，可以考虑是JSON字符串有问题， +// 尤其是在反序列化嵌套的TypeUnion时，如果内部的TypeUnion反序列化失败，错误是不会传递出来的（库的缺陷） func JSONToObjectStreamEx[T any](stream io.Reader) (T, error) { var ret T dec := defaultAPI.NewDecoder(stream) From 3df400be4141eb11f9f974138b7aca5aefc2c75e Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 13 Feb 2025 15:04:05 +0800 Subject: [PATCH 035/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/math2/math.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/math2/math.go b/utils/math2/math.go index db54476..4e6d268 100644 --- a/utils/math2/math.go +++ b/utils/math2/math.go @@ -74,7 +74,7 @@ func SplitN[T constraints.Integer](v T, n int) []T { } // 除法，如果除数为0，则返回默认值 -func DivOrDefault[T constraints.Integer](v, d T, def T) T { +func DivOrDefault[T constraints.Integer | constraints.Float](v, d T, def T) T { if d == 0 { return def } From bb9903e5dd358b0efdb128c10aface9b88766437 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 13 Feb 2025 15:58:05 +0800 Subject: [PATCH 036/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=91=E5=AE=9A=E7=AD=9B=E9=80=89=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/bitmap/bitmap.go | 2 +- pkgs/ioswitch/dag/node.go | 2 +- pkgs/ioswitch/dag/var.go | 10 +++++----- sdks/pcmscheduler/models.go | 13 +++++++++++++ utils/io2/chunked_split.go | 2 +- utils/serder/serder_test.go | 8 ++++---- 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/pkgs/bitmap/bitmap.go b/pkgs/bitmap/bitmap.go index b1c3e63..d655772 100644 --- a/pkgs/bitmap/bitmap.go +++ b/pkgs/bitmap/bitmap.go @@ -23,7 +23,7 @@ func (b *Bitmap64) Weight() int { cnt := 0 for v > 0 { cnt++ - v &= (v - 1) + v &= v - 1 } return cnt } diff --git a/pkgs/ioswitch/dag/node.go b/pkgs/ioswitch/dag/node.go index 6fbe735..d513267 100644 --- a/pkgs/ioswitch/dag/node.go +++ b/pkgs/ioswitch/dag/node.go @@ -98,7 +98,7 @@ func (s *VarSlots[T]) Clear(val *T) { } func (s *VarSlots[T]) RemoveAt(idx int) { - (*s) = lo2.RemoveAt(*s, idx) + *s = lo2.RemoveAt(*s, idx) } func (s *VarSlots[T]) RemoveRange(start int, cnt int) { diff --git a/pkgs/ioswitch/dag/var.go b/pkgs/ioswitch/dag/var.go index 27168a3..5d5989e 100644 --- a/pkgs/ioswitch/dag/var.go +++ b/pkgs/ioswitch/dag/var.go @@ -87,28 +87,28 @@ func (s *DstList) Get(idx int) Node { } func (s *DstList) Add(n Node) int { - (*s) = append((*s), n) + *s = append(*s, n) return len(*s) - 1 } func (s *DstList) Remove(n Node) { for i, e := range *s { if e == n { - (*s) = lo2.RemoveAt((*s), i) + *s = lo2.RemoveAt(*s, i) return } } } func (s *DstList) RemoveAt(idx int) { - lo2.RemoveAt((*s), idx) + lo2.RemoveAt(*s, idx) } func (s *DstList) Resize(size int) { if s.Len() < size { - (*s) = append((*s), make([]Node, size-s.Len())...) + *s = append(*s, make([]Node, size-s.Len())...) } else if s.Len() > size { - (*s) = (*s)[:size] + *s = (*s)[:size] } } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 0a0e14b..8fb7182 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -44,6 +44,9 @@ const ( PreferencePriority = "preference" SpecifyClusterPriority = "specify" + + FailedStatus = "failed" + SuccessStatus = "success" ) type TaskID int64 @@ -253,6 +256,11 @@ type BiasPriority struct { Options []string `json:"options"` } +type TaskMessage struct { + Status string `json:"status"` + Message string `json:"message"` +} + type UploadParams struct { DataType string `json:"dataType"` UploadInfo UploadInfo `json:"uploadInfo"` @@ -406,6 +414,11 @@ type ClusterImage struct { CardType string `json:"cardType"` } +type QueryBindingFilters struct { + Status string `json:"status"` + Name string `json:"name"` +} + type QueryBindingDataParam interface { Noop() } diff --git a/utils/io2/chunked_split.go b/utils/io2/chunked_split.go index b6606c5..07893a0 100644 --- a/utils/io2/chunked_split.go +++ b/utils/io2/chunked_split.go @@ -32,7 +32,7 @@ func ChunkedSplit(stream io.Reader, chunkSize int, streamCount int, opts ...Chun loop: for { for i := 0; i < streamCount; i++ { - var rd int = 0 + var rd = 0 if !eof { var err error rd, err = io.ReadFull(stream, buf) diff --git a/utils/serder/serder_test.go b/utils/serder/serder_test.go index bdb3101..b36c0d1 100644 --- a/utils/serder/serder_test.go +++ b/utils/serder/serder_test.go @@ -591,7 +591,7 @@ func Test_ObjectToJSON2(t *testing.T) { union := types.NewTypeUnion[Base](&St1{}, St2{}) UseTypeUnionExternallyTagged(&union) - var val []Base = []Base{ + var val = []Base{ &St1{Val: "asd", Type: "St1"}, St2{Val: 123, Type: "St2"}, } @@ -607,7 +607,7 @@ func Test_ObjectToJSON2(t *testing.T) { union := types.NewTypeUnion[Base](&St1{}, St2{}) UseTypeUnionInternallyTagged(&union, "Type") - var val []Base = []Base{ + var val = []Base{ &St1{Val: "asd", Type: "St1"}, St2{Val: 123, Type: "St2"}, } @@ -623,7 +623,7 @@ func Test_ObjectToJSON2(t *testing.T) { union := types.NewTypeUnion[Base](&St1{}, St2{}) UseTypeUnionExternallyTagged(&union) - var val []Base = []Base{ + var val = []Base{ nil, } data, err := ObjectToJSONEx(val) @@ -638,7 +638,7 @@ func Test_ObjectToJSON2(t *testing.T) { union := types.NewTypeUnion[Base](&St1{}, St2{}) UseTypeUnionInternallyTagged(&union, "Type") - var val []Base = []Base{ + var val = []Base{ nil, } data, err := ObjectToJSONEx(val) From eb37cb3e5da53b19be60e569b1b84b35ceb82065 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 13 Feb 2025 17:37:39 +0800 Subject: [PATCH 037/155] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=91=E5=AE=9A=E5=BC=82=E5=B8=B8=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 1d05570..dca430c 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -144,9 +144,7 @@ func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { return &codeResp.Data, nil } - //return nil, codeResp.ToError() - println(codeResp.Message) - return nil, err + return nil, fmt.Errorf("error: %s", codeResp.Message) } return nil, fmt.Errorf("unknow response content type: %s", contType) From 28882af18fd8db9537e05aa3fe32e85f041c6199 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 14 Feb 2025 11:00:56 +0800 Subject: [PATCH 038/155] =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=80=9A=E7=94=A8?= =?UTF-8?q?=E7=9A=84S3=E5=AD=98=E5=82=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/public_storage.go | 15 +++++++++++++++ sdks/storage/storage.go | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/sdks/storage/public_storage.go b/sdks/storage/public_storage.go index 59d1341..761a6a8 100644 --- a/sdks/storage/public_storage.go +++ b/sdks/storage/public_storage.go @@ -15,6 +15,7 @@ type PublicStoreConfig interface { var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[PublicStoreConfig]( (*LocalPublicStorage)(nil), + (*S3PublicStorage)(nil), )), "type") type LocalPublicStorage struct { @@ -31,3 +32,17 @@ func (s *LocalPublicStorage) GetPublicStoreType() string { func (s *LocalPublicStorage) String() string { return fmt.Sprintf("Local[LoadBase=%v]", s.LoadBase) } + +type S3PublicStorage struct { + serder.Metadata `union:"S3"` + Type string `json:"type"` + LoadBase string `json:"loadBase"` +} + +func (s *S3PublicStorage) GetPublicStoreType() string { + return "S3" +} + +func (s *S3PublicStorage) String() string { + return fmt.Sprintf("S3[LoadBase=%v]", s.LoadBase) +} diff --git a/sdks/storage/storage.go b/sdks/storage/storage.go index cfbca6d..9f1fffd 100644 --- a/sdks/storage/storage.go +++ b/sdks/storage/storage.go @@ -44,6 +44,7 @@ var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Storage (*OSSType)(nil), (*COSType)(nil), (*EFileType)(nil), + (*S3Type)(nil), )), "type") // 多种存储服务的混合存储服务。需谨慎选择存储服务的组合，避免出Bug @@ -149,3 +150,22 @@ func (a *EFileType) GetStorageType() string { func (a *EFileType) String() string { return "EFile" } + +// 通用的S3协议的存储服务 +type S3Type struct { + serder.Metadata `union:"S3"` + Type string `json:"type"` + Region string `json:"region"` + AK string `json:"accessKeyId"` + SK string `json:"secretAccessKey"` + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` +} + +func (a *S3Type) GetStorageType() string { + return "S3" +} + +func (a *S3Type) String() string { + return "S3" +} From 725f244f245278b0dea9428dd0a80941f6abfc31 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 17 Feb 2025 09:45:50 +0800 Subject: [PATCH 039/155] =?UTF-8?q?=E7=BB=91=E5=AE=9A=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=88=9B=E5=BB=BA=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 7 ++++--- sdks/uploader/models.go | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 8fb7182..6f5ff62 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -364,7 +364,7 @@ type DatasetBinding struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` Description string `json:"description"` Category string `json:"category"` - PackageIDs []cdssdk.PackageID `json:"packageIDs"` + PackageID cdssdk.PackageID `json:"packageID"` } type ModelBinding struct { @@ -375,9 +375,10 @@ type ModelBinding struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` Description string `json:"description"` Category string `json:"category"` + ModelType string `json:"modelType"` Env string `json:"env"` Version string `json:"version"` - PackageIDs []cdssdk.PackageID `json:"packageIDs"` + PackageID cdssdk.PackageID `json:"packageID"` } type CodeBinding struct { @@ -403,7 +404,7 @@ type ImageBinding struct { Architecture string `json:"architecture"` ResourceType string `json:"resourceType"` Tags []string `json:"tags"` - PackageIDs []cdssdk.PackageID `json:"packageIDs"` + PackageID cdssdk.PackageID `json:"packageID"` } type ClusterImage struct { diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 6babe86..40f9d43 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -46,6 +46,7 @@ type BindingAccessData struct { AccessLevel string `gorm:"column:access_level" json:"accessLevel"` ApplicantID cdssdk.UserID `gorm:"column:applicant_id" json:"applicantID"` Status string `gorm:"column:status" json:"status"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } type BindingDetail struct { @@ -58,6 +59,7 @@ type BindingDetail struct { Packages []Package `json:"packages"` Status string `json:"status"` AccessLevel string `json:"accessLevel"` + CreateTime time.Time `json:"createTime"` } func (Binding) TableName() string { From 5d96ee473cfc39d2e5a616a932e2ae13ffb8c9b6 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 17 Feb 2025 16:16:49 +0800 Subject: [PATCH 040/155] merge --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 16bebdb..4e0d2a0 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect diff --git a/go.sum b/go.sum index 60a4de8..c51be4b 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= From 5af81e1301366acfbedd32ec90ae27ab220ff5f7 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 18 Feb 2025 14:57:03 +0800 Subject: [PATCH 041/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 6f5ff62..fae6c6a 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -47,6 +47,9 @@ const ( FailedStatus = "failed" SuccessStatus = "success" + + Query = "query" + Delete = "delete" ) type TaskID int64 @@ -284,6 +287,7 @@ type LocalUploadInfo struct { Type string `json:"type"` LocalPath string `json:"localPath"` ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` + PackageID cdssdk.PackageID `json:"packageID"` } type RemoteUploadInfo struct { @@ -388,7 +392,7 @@ type CodeBinding struct { Name string `json:"name"` ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` Description string `json:"description"` - ImageID string `json:"imageID"` + ImageID int64 `json:"imageID"` ObjectID cdssdk.ObjectID `json:"objectID"` FilePath string `json:"filePath"` PackageID cdssdk.PackageID `json:"packageID"` From 0dc4b2374644edf5195f5fc047ba3bb76dbb6faf Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 19 Feb 2025 14:49:00 +0800 Subject: [PATCH 042/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=85=8B=E9=9A=86?= =?UTF-8?q?=E5=92=8C=E6=9F=A5=E8=AF=A2package=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 145 ++++++++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 59 deletions(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 40f9d43..5cfa55b 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -90,72 +90,99 @@ func (Cluster) TableName() string { } type Package struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:create_time" json:"createTime"` - Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` - UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` - Versions []PackageVersion `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` + Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` + UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` + Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` } type PackageDAO struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:create_time" json:"createTime"` - UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 - Versions []PackageVersion `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` - UploadPriority string `gorm:"column:upload_priority" json:"uploadPriority"` -} - -type PackageVersion struct { - ParentPackageID cdssdk.PackageID `gorm:"column:parent_package_id" json:"parentPackageID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - Version int64 `gorm:"column:package_version" json:"version"` -} - -func (PackageVersion) TableName() string { - return "packageVersion" // 确保和数据库中的表名一致 -} - -//type PackageScheduleType interface { -// Noop() -//} -// -//var PackageScheduleTypeUnion = types.NewTypeUnion[PackageScheduleType]( -// (*PackagePreferencesSchedule)(nil), -// (*PackageSpecifyClusterSchedule)(nil), -//) -// -//var _ = serder.UseTypeUnionInternallyTagged(&PackageScheduleTypeUnion, "type") -// -//type PackageScheduleBase struct{} -// -//func (d *PackageScheduleBase) Noop() {} -// -//type PackagePreferencesSchedule struct { -// serder.Metadata `union:"region"` -// PackageScheduleBase -// Type string `json:"type"` -// a sch.Preferences + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` + UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 + Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` + UploadPriority string `gorm:"column:upload_priority" json:"uploadPriority"` +} + +type PackageCloneDAO struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + ParentPackageID cdssdk.PackageID `gorm:"column:parent_package_id" json:"parentPackageID"` + ClonePackageID cdssdk.PackageID `gorm:"column:clone_package_id" json:"clonePackageID"` + Name string `gorm:"column:name" json:"name"` + Description string `gorm:"column:description" json:"description"` + BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` + ImageID string `gorm:"column:image_id" json:"imageID"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` +} + +func (PackageCloneDAO) TableName() string { + return "packageClone" // 确保和数据库中的表名一致 +} + +type PackageCloneParam struct { + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + PackageName string `json:"packageName" binding:"required"` + BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` + Name string `json:"name"` + Description string `json:"description"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + ClusterID schsdk.ClusterID `json:"clusterID"` + ParentImageID schsdk.ImageID `json:"parentImageID"` + ImageID string `json:"imageID"` +} + +type PackageCloneVO struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + ParentPackageID cdssdk.PackageID `gorm:"column:parent_package_id" json:"parentPackageID"` + ClonePackageID cdssdk.PackageID `gorm:"column:clone_package_id" json:"clonePackageID"` + Name string `gorm:"column:name" json:"name"` + Description string `gorm:"column:description" json:"description"` + BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` + ImageID string `gorm:"column:image_id" json:"imageID"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"` +} + +type ClusterMapping struct { + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + ClusterName string `gorm:"column:cluster_name" json:"clusterName"` + StorageID cdssdk.StorageID `gorm:"column:storage_id" json:"storageID"` + StorageType string `gorm:"column:storage_type" json:"storageType"` + StoragePath string `gorm:"column:storage_path" json:"storagePath"` +} + +func (ClusterMapping) TableName() string { + return "ClusterMapping" +} + +//type PackageCloneClusterDAO struct { +// ID DataID `gorm:"column:id" json:"ID"` +// ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` +// ClusterName string `gorm:"column:cluster_name" json:"clusterName"` //} // -//type PackageSpecifyClusterSchedule struct { -// serder.Metadata `union:"region"` -// PackageScheduleBase -// Type string `json:"type"` +//type PackageCloneCluster struct { +// ClusterID schsdk.ClusterID `json:"clusterID"` +// ClusterName string `json:"clusterName"` //} type ScheduleTarget interface { From 9c69c9d910bf92c3fbf3543f0f930ab709ba77de Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 19 Feb 2025 15:22:43 +0800 Subject: [PATCH 043/155] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=95=9C=E5=83=8F?= =?UTF-8?q?=E7=BB=91=E5=AE=9A=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index fae6c6a..0505232 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -398,17 +398,28 @@ type CodeBinding struct { PackageID cdssdk.PackageID `json:"packageID"` } +//type ImageBinding struct { +// serder.Metadata `union:"image"` +// DataBindingBase +// Type string `json:"type"` +// Name string `json:"name"` +// ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` +// Description string `json:"description"` +// Architecture string `json:"architecture"` +// ResourceType string `json:"resourceType"` +// Tags []string `json:"tags"` +// PackageID cdssdk.PackageID `json:"packageID"` +//} + type ImageBinding struct { serder.Metadata `union:"image"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Architecture string `json:"architecture"` - ResourceType string `json:"resourceType"` - Tags []string `json:"tags"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + ID int64 `json:"id"` + Name string `json:"name"` + IDType string `json:"idType"` + ImageID string `json:"imageID"` + ClusterID schsdk.ClusterID `json:"clusterID"` } type ClusterImage struct { From 4c65ac188d9590efa83220ab7c88998a815f073d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 19 Feb 2025 16:42:46 +0800 Subject: [PATCH 044/155] =?UTF-8?q?=E5=88=9B=E5=BB=BA=E7=AE=97=E6=B3=95?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E6=96=B0=E5=A2=9Etype=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 0505232..9a66505 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -50,6 +50,9 @@ const ( Query = "query" Delete = "delete" + + ChildrenType = "children" + ParentType = "parent" ) type TaskID int64 From 998c2e98095789222ca7a04b2a0966e6175a82e7 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 21 Feb 2025 14:30:05 +0800 Subject: [PATCH 045/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 4 ++++ sdks/uploader/models.go | 23 +++++++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 9a66505..84c7840 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -53,6 +53,10 @@ const ( ChildrenType = "children" ParentType = "parent" + + PlatformSugon = "sugon" + PlatformOpenI = "openi" + PlatformModelArts = "modelarts" ) type TaskID int64 diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 5cfa55b..5921a1a 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -25,14 +25,14 @@ func (BlockChain) TableName() string { } type Binding struct { - ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - Name string `gorm:"column:name" json:"Name"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` - Content string `gorm:"column:content" json:"Content"` - AccessLevel string `gorm:"column:access_level" json:"accessLevel"` - CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + //JsonData string `gorm:"column:json_data" json:"jsonData"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } type BindingAccessData struct { @@ -66,6 +66,13 @@ func (Binding) TableName() string { return "BindingData" // 确保和数据库中的表名一致 } +type BindingCluster struct { + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + ClusterID ClusterID `gorm:"column:cluster_id" json:"clusterID"` + Status string `gorm:"column:status" json:"status"` + JsonData string `gorm:"column:json_data" json:"jsonData"` +} + type Folder struct { PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` Path string `gorm:"column:path_name" json:"path"` From be552f65ce9fd2c9256a80dfc616d874f8ec6229 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 21 Feb 2025 15:37:19 +0800 Subject: [PATCH 046/155] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AD=90=E7=AE=97?= =?UTF-8?q?=E6=B3=95=E6=8E=A5=E5=8F=A3=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 13 +++++++------ sdks/uploader/models.go | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 84c7840..101e50b 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -294,17 +294,18 @@ type LocalUploadInfo struct { Type string `json:"type"` LocalPath string `json:"localPath"` ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` - PackageID cdssdk.PackageID `json:"packageID"` + //PackageID cdssdk.PackageID `json:"packageID"` } type RemoteUploadInfo struct { serder.Metadata `union:"url"` UploadInfoBase - Type string `json:"type"` - Url string `json:"url"` - DataName string `json:"dataName"` - Cluster schsdk.ClusterID `json:"clusterID"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Url string `json:"url"` + Branch string `json:"branch"` + DataName string `json:"dataName"` + Cluster schsdk.ClusterID `json:"clusterID"` + //PackageID cdssdk.PackageID `json:"packageID"` } type UploadInfoBase struct{} diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 5921a1a..1fe59d1 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -144,9 +144,9 @@ func (PackageCloneDAO) TableName() string { } type PackageCloneParam struct { - PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - PackageName string `json:"packageName" binding:"required"` - BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + PackageName string `json:"packageName" binding:"required"` + //BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` Name string `json:"name"` Description string `json:"description"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` From 1b1e210001b12aab88fa328313c7f0f454687be8 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 21 Feb 2025 17:22:24 +0800 Subject: [PATCH 047/155] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 101e50b..59af97e 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -294,7 +294,6 @@ type LocalUploadInfo struct { Type string `json:"type"` LocalPath string `json:"localPath"` ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` - //PackageID cdssdk.PackageID `json:"packageID"` } type RemoteUploadInfo struct { @@ -305,7 +304,6 @@ type RemoteUploadInfo struct { Branch string `json:"branch"` DataName string `json:"dataName"` Cluster schsdk.ClusterID `json:"clusterID"` - //PackageID cdssdk.PackageID `json:"packageID"` } type UploadInfoBase struct{} From aa1090544599ff3b6925eb6a244330aa4d4859db Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 24 Feb 2025 15:34:51 +0800 Subject: [PATCH 048/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=95=9C=E5=83=8F?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 59af97e..d0ab2ff 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -5,6 +5,7 @@ import ( schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/serder" + "time" ) type ResourceType string @@ -428,12 +429,33 @@ type ImageBinding struct { ClusterID schsdk.ClusterID `json:"clusterID"` } +type Image struct { + ImageID schsdk.ImageID `json:"imageID" gorm:"column:ImageID;primaryKey"` + Name string `json:"name" gorm:"column:Name"` + CreateTime time.Time `json:"createTime" gorm:"column:CreateTime"` + ClusterImage []ClusterImage `gorm:"foreignKey:image_id;references:ImageID" json:"clusterImages"` +} + type ClusterImage struct { - ID int64 `json:"id"` - Name string `json:"name"` - ClusterID schsdk.ClusterID `json:"clusterID"` - ClusterImageID string `json:"clusterImageID"` - CardType string `json:"cardType"` + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + OriginImageType string `gorm:"column:origin_image_type" json:"originImageType"` + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + OriginImageName string `gorm:"column:origin_image_name" json:"originImageName"` + ClusterImageCard []ClusterImageCard `gorm:"foreignKey:origin_image_id;references:origin_image_id" json:"cards"` +} + +func (ClusterImage) TableName() string { + return "clusterImage" +} + +type ClusterImageCard struct { + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + Card string `gorm:"column:card" json:"card"` +} + +func (ClusterImageCard) TableName() string { + return "clusterImageCard" } type QueryBindingFilters struct { From 83fb1a0d585a164ebe440472447a4be191f21a5f Mon Sep 17 00:00:00 2001 From: qiwang <1364512070@qq.com> Date: Wed, 26 Feb 2025 09:36:35 +0800 Subject: [PATCH 049/155] feat: Add image filtering in resource recommendations --- sdks/pcmscheduler/models.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index d0ab2ff..af991f5 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -17,6 +17,8 @@ const ( ResourceTypeMLU ResourceType = "MLU" ResourceTypeStorage ResourceType = "STORAGE" ResourceTypeMemory ResourceType = "MEMORY" + ResourceTypeDCU ResourceType = "DCU" + ResourceTypeGCU ResourceType = "GCU" Split = "/" @@ -222,6 +224,7 @@ type ResourceRange struct { CPU Range `json:"cpu"` Memory Range `json:"memory"` Storage Range `json:"storage"` + Ids []string `json:"ids"` } type Range struct { From 63cf68a413212a582c1be1f07184bef756d77b22 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 26 Feb 2025 15:19:55 +0800 Subject: [PATCH 050/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=91=E5=AE=9A=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 12 +++- sdks/pcmscheduler/models.go | 21 ++++--- sdks/scheduler/models.go | 4 +- sdks/uploader/models.go | 31 ++++++++--- sdks/uploader/uploader.go | 107 ++++++++++++++++++------------------ 5 files changed, 100 insertions(+), 75 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index dca430c..ace7fac 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -64,6 +64,11 @@ type CreateJobReq struct { DataDistribute DataDistribute `json:"dataDistributes"` } +type CommonJsonData struct { + ID string `json:"id"` + Name string `json:"name"` +} + type DataDistribute struct { Dataset []DatasetDistribute `json:"dataset"` Code []CodeDistribute `json:"code"` @@ -90,9 +95,10 @@ type CodeDistribute struct { } type ImageDistribute struct { - DataName string `json:"dataName"` - PackageID cdssdk.PackageID `json:"packageID"` - Clusters []DataDetail `json:"clusters"` + DataName string `json:"dataName"` + //PackageID cdssdk.PackageID `json:"packageID"` + ImageID schsdk.ImageID `json:"imageID"` + Clusters []DataDetail `json:"clusters"` } type ModelDistribute struct { diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index d0ab2ff..4007f4a 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -58,9 +58,13 @@ const ( PlatformSugon = "sugon" PlatformOpenI = "openi" PlatformModelArts = "modelarts" + + URL = "url" + ID = "id" ) type TaskID int64 +type DataID int64 type ClusterDetail struct { // 集群ID @@ -395,14 +399,15 @@ type ModelBinding struct { type CodeBinding struct { serder.Metadata `union:"code"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - ImageID int64 `json:"imageID"` - ObjectID cdssdk.ObjectID `json:"objectID"` - FilePath string `json:"filePath"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + //ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Description string `json:"description"` + ImageID schsdk.ImageID `json:"imageID"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + //FilePath string `json:"filePath"` + PackageID cdssdk.PackageID `json:"packageID"` } //type ImageBinding struct { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 906796d..8992f5b 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -115,7 +115,7 @@ type PCMJobRuntimeInfo struct { //} type JobResource interface { - Noop2() + Noop() } var JobResourceTypeUnion = types.NewTypeUnion[JobResource]( @@ -132,7 +132,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type") type JobResourceBase struct{} -func (d *JobResourceBase) Noop2() {} +func (d *JobResourceBase) Noop() {} type CPU struct { serder.Metadata `union:"CPU"` diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 1fe59d1..4d28dc7 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -25,14 +25,24 @@ func (BlockChain) TableName() string { } type Binding struct { - ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - Name string `gorm:"column:name" json:"Name"` - DataType string `gorm:"column:data_type" json:"dataType"` - //JsonData string `gorm:"column:json_data" json:"jsonData"` - Content string `gorm:"column:content" json:"Content"` - AccessLevel string `gorm:"column:access_level" json:"accessLevel"` - CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` +} + +type BindingDAO struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + BindingCluster []BindingCluster `gorm:"foreignKey:binding_id;references:id" json:"bindingCluster"` } type BindingAccessData struct { @@ -73,6 +83,10 @@ type BindingCluster struct { JsonData string `gorm:"column:json_data" json:"jsonData"` } +func (BindingCluster) TableName() string { + return "bindingCluster" // 确保和数据库中的表名一致 +} + type Folder struct { PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` Path string `gorm:"column:path_name" json:"path"` @@ -136,6 +150,7 @@ type PackageCloneDAO struct { ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` ImageID string `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go index 7a99b2c..54f17df 100644 --- a/sdks/uploader/uploader.go +++ b/sdks/uploader/uploader.go @@ -3,7 +3,6 @@ package uploadersdk import ( "fmt" "gitlink.org.cn/cloudream/common/pkgs/types" - sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" "gitlink.org.cn/cloudream/common/utils/http2" @@ -12,65 +11,65 @@ import ( "strings" ) -type DataScheduleReq struct { - PackageID cdssdk.PackageID `json:"packageID"` - DataType string `json:"dataType"` - ScheduleTarget ScheduleTarget `json:"scheduleTarget"` -} +//type DataScheduleReq struct { +// PackageID cdssdk.PackageID `json:"packageID"` +// DataType string `json:"dataType"` +// ScheduleTarget ScheduleTarget `json:"scheduleTarget"` +//} //type DataScheduleResp struct { // Results []sch.DataScheduleResult `json:"data"` //} -type TmpDataScheduleResult struct { - Cluster sch.DataDetail `json:"cluster"` - PackageID cdssdk.PackageID `json:"packageID"` - Status bool `json:"status"` - Msg string `json:"msg"` -} - -func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule") - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - println(resp.Body) - - contType := resp.Header.Get("Content-Type") - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[[]TmpDataScheduleResult] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return nil, fmt.Errorf("parsing response: %w", err) - } - - if codeResp.Code == ResponseCodeOK { - var results []sch.DataScheduleResult - for _, tmpResult := range codeResp.Data { - result := sch.DataScheduleResult{ - PackageID: tmpResult.PackageID, - Status: tmpResult.Status, - Msg: tmpResult.Msg, - Clusters: []sch.DataDetail{ - tmpResult.Cluster, - }, - } - results = append(results, result) - } - return results, nil - } - - return nil, codeResp.ToError() - } +//type TmpDataScheduleResult struct { +// Cluster sch.DataDetail `json:"cluster"` +// PackageID cdssdk.PackageID `json:"packageID"` +// Status bool `json:"status"` +// Msg string `json:"msg"` +//} - return nil, fmt.Errorf("unknow response content type: %s", contType) -} +//func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) { +// targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule") +// if err != nil { +// return nil, err +// } +// +// resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ +// Body: req, +// }) +// if err != nil { +// return nil, err +// } +// println(resp.Body) +// +// contType := resp.Header.Get("Content-Type") +// if strings.Contains(contType, http2.ContentTypeJSON) { +// var codeResp response[[]TmpDataScheduleResult] +// if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { +// return nil, fmt.Errorf("parsing response: %w", err) +// } +// +// if codeResp.Code == ResponseCodeOK { +// var results []sch.DataScheduleResult +// for _, tmpResult := range codeResp.Data { +// result := sch.DataScheduleResult{ +// PackageID: tmpResult.PackageID, +// Status: tmpResult.Status, +// Msg: tmpResult.Msg, +// Clusters: []sch.DataDetail{ +// tmpResult.Cluster, +// }, +// } +// results = append(results, result) +// } +// return results, nil +// } +// +// return nil, codeResp.ToError() +// } +// +// return nil, fmt.Errorf("unknow response content type: %s", contType) +//} type UploadReq struct { DataType string `json:"dataType"` From bea84c8eb4027b1d021c684bd070501b899c2d9a Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 26 Feb 2025 16:04:37 +0800 Subject: [PATCH 051/155] =?UTF-8?q?=E5=AD=90=E7=AE=97=E6=B3=95=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 4d28dc7..68c5d8c 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -149,7 +149,7 @@ type PackageCloneDAO struct { BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` - ImageID string `gorm:"column:image_id" json:"imageID"` + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } @@ -167,7 +167,7 @@ type PackageCloneParam struct { BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `json:"clusterID"` ParentImageID schsdk.ImageID `json:"parentImageID"` - ImageID string `json:"imageID"` + ImageID schsdk.ImageID `json:"imageID"` } type PackageCloneVO struct { @@ -180,6 +180,7 @@ type PackageCloneVO struct { ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` ImageID string `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:created_at" json:"createTime"` ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"` } From 30d1aee0a9e8a26087109660c9ed21eadd7870fe Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 26 Feb 2025 17:27:59 +0800 Subject: [PATCH 052/155] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=90=8C=E6=AD=A5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/sync2/bucket_pool.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/utils/sync2/bucket_pool.go b/utils/sync2/bucket_pool.go index 2551a42..f5fa633 100644 --- a/utils/sync2/bucket_pool.go +++ b/utils/sync2/bucket_pool.go @@ -7,6 +7,7 @@ type BucketPool[T any] struct { filled []T emptyCond *sync.Cond filledCond *sync.Cond + closed bool } func NewBucketPool[T any]() *BucketPool[T] { @@ -20,6 +21,11 @@ func (p *BucketPool[T]) GetEmpty() (T, bool) { p.emptyCond.L.Lock() defer p.emptyCond.L.Unlock() + if p.closed { + var t T + return t, false + } + if len(p.empty) == 0 { p.emptyCond.Wait() } @@ -46,6 +52,11 @@ func (p *BucketPool[T]) GetFilled() (T, bool) { p.filledCond.L.Lock() defer p.filledCond.L.Unlock() + if p.closed { + var t T + return t, false + } + if len(p.filled) == 0 { p.filledCond.Wait() } @@ -68,7 +79,17 @@ func (p *BucketPool[T]) PutFilled(t T) { p.filledCond.Signal() } -func (p *BucketPool[T]) WakeUpAll() { +func (p *BucketPool[T]) Close() { + // 在两个锁中分别关闭，防止变量可见性问题 + + p.emptyCond.L.Lock() + p.closed = true + p.emptyCond.L.Unlock() + + p.filledCond.L.Lock() + p.closed = true + p.filledCond.L.Unlock() + p.emptyCond.Broadcast() p.filledCond.Broadcast() } From 13658da85d310f824746037ffcb0f34a4a6526fb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 27 Feb 2025 10:35:42 +0800 Subject: [PATCH 053/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=8F=90=E4=BA=A4=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 7 ++++--- sdks/scheduler/models.go | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index ace7fac..8bbdd4e 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -78,8 +78,9 @@ type DataDistribute struct { type DataDetail struct { ClusterID schsdk.ClusterID `json:"clusterID"` - StorageID cdssdk.StorageID `json:"storageID"` - JsonData string `json:"jsonData"` + //StorageID cdssdk.StorageID `json:"storageID"` + StorageID cdssdk.StorageID + JsonData string `json:"jsonData"` } type DatasetDistribute struct { @@ -97,7 +98,7 @@ type CodeDistribute struct { type ImageDistribute struct { DataName string `json:"dataName"` //PackageID cdssdk.PackageID `json:"packageID"` - ImageID schsdk.ImageID `json:"imageID"` + ImageID schsdk.ImageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 8992f5b..cbf0ae7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -99,9 +99,11 @@ type JobResources struct { } type ClusterInfo struct { - ClusterID ClusterID `json:"clusterID"` - Resources []JobResource `json:"resources"` - Runtime PCMJobRuntimeInfo `json:"runtime"` + ClusterID ClusterID `json:"clusterID"` + Resources []JobResource `json:"resources"` + //Files JobFilesInfo `json:"files"` + Code JobFileInfo `json:"code"` + Runtime PCMJobRuntimeInfo `json:"runtime"` } type PCMJobRuntimeInfo struct { From fb1cc453b0054726d41c24f84eea2f47aef68c09 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 27 Feb 2025 15:08:12 +0800 Subject: [PATCH 054/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 47 +++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 68c5d8c..2a92a89 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -127,12 +127,12 @@ type Package struct { } type PackageDAO struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:create_time" json:"createTime"` UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 @@ -148,10 +148,10 @@ type PackageCloneDAO struct { Description string `gorm:"column:description" json:"description"` BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` - ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` - ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + //ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } func (PackageCloneDAO) TableName() string { @@ -159,14 +159,12 @@ func (PackageCloneDAO) TableName() string { } type PackageCloneParam struct { - PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - PackageName string `json:"packageName" binding:"required"` - //BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + PackageName string `json:"packageName" binding:"required"` Name string `json:"name"` Description string `json:"description"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `json:"clusterID"` - ParentImageID schsdk.ImageID `json:"parentImageID"` ImageID schsdk.ImageID `json:"imageID"` } @@ -178,11 +176,11 @@ type PackageCloneVO struct { Description string `gorm:"column:description" json:"description"` BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` - ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` - ImageID string `gorm:"column:image_id" json:"imageID"` - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:created_at" json:"createTime"` - ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"` + //ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` + ImageID string `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"` } type ClusterMapping struct { @@ -197,17 +195,6 @@ func (ClusterMapping) TableName() string { return "ClusterMapping" } -//type PackageCloneClusterDAO struct { -// ID DataID `gorm:"column:id" json:"ID"` -// ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` -// ClusterName string `gorm:"column:cluster_name" json:"clusterName"` -//} -// -//type PackageCloneCluster struct { -// ClusterID schsdk.ClusterID `json:"clusterID"` -// ClusterName string `json:"clusterName"` -//} - type ScheduleTarget interface { Noop() } From 7a9b0184d7a94cea97d72502196c0cadc88aea62 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 27 Feb 2025 15:09:47 +0800 Subject: [PATCH 055/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/utils.go | 13 +++++++++ utils/sync2/token_pool.go | 60 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 sdks/storage/utils.go create mode 100644 utils/sync2/token_pool.go diff --git a/sdks/storage/utils.go b/sdks/storage/utils.go new file mode 100644 index 0000000..39268dd --- /dev/null +++ b/sdks/storage/utils.go @@ -0,0 +1,13 @@ +package cdssdk + +import ( + "strings" +) + +func JoinObjectPath(comps ...string) string { + return strings.Join(comps, ObjectPathSeparator) +} + +func SplitObjectPath(pat string) []string { + return strings.Split(pat, ObjectPathSeparator) +} diff --git a/utils/sync2/token_pool.go b/utils/sync2/token_pool.go new file mode 100644 index 0000000..3fda84b --- /dev/null +++ b/utils/sync2/token_pool.go @@ -0,0 +1,60 @@ +package sync2 + +import "sync" + +type TokenPool[T any] struct { + tokens []T + cond *sync.Cond + closed bool +} + +func NewTokenPool[T any]() *TokenPool[T] { + return &TokenPool[T]{ + tokens: make([]T, 0), + cond: sync.NewCond(new(sync.Mutex)), + } +} + +func NewTokenPoolWithTokens[T any](tokens ...T) *TokenPool[T] { + b := NewTokenPool[T]() + b.tokens = append(b.tokens, tokens...) + return b +} + +func (b *TokenPool[T]) Take() (T, bool) { + b.cond.L.Lock() + defer b.cond.L.Unlock() + var ret T + + if b.closed { + return ret, false + } + + if len(b.tokens) == 0 { + b.cond.Wait() + + if len(b.tokens) == 0 { + return ret, false + } + } + + ret = b.tokens[0] + b.tokens = b.tokens[1:] + + return ret, true +} + +func (b *TokenPool[T]) Put(t T) { + b.cond.L.Lock() + b.tokens = append(b.tokens, t) + b.cond.L.Unlock() + b.cond.Signal() +} + +func (b *TokenPool[T]) Close() { + b.cond.L.Lock() + b.closed = true + b.cond.L.Unlock() + + b.cond.Broadcast() +} From 1e2e2e4b0d56c1448e7fa9ceb77092e73af886a0 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 28 Feb 2025 09:52:29 +0800 Subject: [PATCH 056/155] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E5=88=97=E8=A1=A8=E6=8E=A5=E5=8F=A3=E6=94=AF=E6=8C=81=E5=8F=AA?= =?UTF-8?q?=E9=9D=9E=E9=80=92=E5=BD=92=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index e3486fb..e5a8f61 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -28,13 +28,16 @@ func (c *Client) Object() *ObjectService { const ObjectListPathByPath = "/object/listByPath" type ObjectListByPath struct { - UserID cdssdk.UserID `form:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" binding:"required"` - Path string `form:"path"` // 允许为空字符串 - IsPrefix bool `form:"isPrefix"` + UserID cdssdk.UserID `form:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required"` + Path string `form:"path"` // 允许为空字符串 + IsPrefix bool `form:"isPrefix"` + NoRecursive bool `form:"noRecursive"` // 仅当isPrefix为true时有效，表示仅查询直接属于Prefix下的对象，对于更深的对象，返回它们的公共前缀 + } type ObjectListByPathResp struct { - Objects []cdssdk.Object `json:"objects"` + CommonPrefixes []string `json:"commonPrefixes"` // 仅在IsPrefix为true且NoRecursive为true时有效，包含更深层对象的shared prefix + Objects []cdssdk.Object `json:"objects"` // 如果IsPrefix为true且NoRecursive为false，则返回所有匹配的对象，否则只返回直接属于Prefix下的对象 } func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { From 91563ca94108529e6323f03fef11acaeb30a30eb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 28 Feb 2025 13:50:51 +0800 Subject: [PATCH 057/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 4007f4a..32c55f7 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -61,6 +61,8 @@ const ( URL = "url" ID = "id" + + Startup = "startup" ) type TaskID int64 From 1016cdffd5c44a581f6349be306f11ab9826359e Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 28 Feb 2025 14:50:47 +0800 Subject: [PATCH 058/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=AD=90=E7=AE=97=E6=B3=95=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 2a92a89..c4126bc 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -66,7 +66,7 @@ type BindingDetail struct { SSOId string `json:"ssoID"` Name string `json:"Name"` Info sch.DataBinding `json:"info"` - Packages []Package `json:"packages"` + Package Package `json:"package"` Status string `json:"status"` AccessLevel string `json:"accessLevel"` CreateTime time.Time `json:"createTime"` From fc44199f4233dc92e29087d33ae5fd5d33f4ea2e Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 3 Mar 2025 17:19:12 +0800 Subject: [PATCH 059/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=86=E6=AE=B5?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=86=97=E4=BD=99=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/models.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sdks/storage/models.go b/sdks/storage/models.go index 7fcfc4d..293618a 100644 --- a/sdks/storage/models.go +++ b/sdks/storage/models.go @@ -40,6 +40,7 @@ var RedundancyUnion = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTyp (*ECRedundancy)(nil), (*LRCRedundancy)(nil), (*SegmentRedundancy)(nil), + (*MultipartUploadRedundancy)(nil), )), "type") type NoneRedundancy struct { @@ -229,6 +230,17 @@ func (b *SegmentRedundancy) CalcSegmentRange(start int64, end *int64) (segIdxSta return } +type MultipartUploadRedundancy struct { + serder.Metadata `union:"multipartUpload"` + Type string `json:"type"` +} + +func NewMultipartUploadRedundancy() *MultipartUploadRedundancy { + return &MultipartUploadRedundancy{ + Type: "multipartUpload", + } +} + type User struct { UserID UserID `gorm:"column:UserID; primaryKey; type:bigint" json:"userID"` Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` From e888412c58a6d363397cd6e17ee4c3ceb56ef9b0 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 4 Mar 2025 11:16:30 +0800 Subject: [PATCH 060/155] =?UTF-8?q?=E6=96=B0=E5=A2=9Etoken?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 10 +++++++-- sdks/pcmscheduler/models.go | 42 +++++++++++++++++++------------------ sdks/uploader/models.go | 1 + 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 8bbdd4e..6f6a366 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -120,7 +120,7 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } -func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "/createTask") if err != nil { return nil, err @@ -135,6 +135,9 @@ func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, + Header: map[string]string{ + "Authorization": token, + }, }) if err != nil { return nil, err @@ -176,7 +179,7 @@ type DataScheduleResults struct { Results []DataScheduleResult `json:"results"` } -func (c *Client) RunJob(req RunJobReq) error { +func (c *Client) RunJob(req RunJobReq, token string) error { targetUrl, err := url.JoinPath(c.baseURL, "runTask") if err != nil { return err @@ -184,6 +187,9 @@ func (c *Client) RunJob(req RunJobReq) error { resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, + Header: map[string]string{ + "Authorization": token, + }, }) if err != nil { return err diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 32c55f7..4c4a490 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -376,40 +376,42 @@ func (d *DataBindingBase) Noop() {} type DatasetBinding struct { serder.Metadata `union:"dataset"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` } type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` } type CodeBinding struct { serder.Metadata `union:"code"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - //ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Type string `json:"type"` + Name string `json:"name"` ClusterID schsdk.ClusterID `json:"clusterID"` Description string `json:"description"` ImageID schsdk.ImageID `json:"imageID"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` - //FilePath string `json:"filePath"` - PackageID cdssdk.PackageID `json:"packageID"` + PackageID cdssdk.PackageID `json:"packageID"` + // 当集群为openi的时候，需要传入分支 + Branch string `json:"branch"` } //type ImageBinding struct { diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index c4126bc..7a9879a 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -80,6 +80,7 @@ type BindingCluster struct { BindingID DataID `gorm:"column:binding_id" json:"bindingID"` ClusterID ClusterID `gorm:"column:cluster_id" json:"clusterID"` Status string `gorm:"column:status" json:"status"` + Param string `gorm:"column:param" json:"Param"` JsonData string `gorm:"column:json_data" json:"jsonData"` } From 3a90a6cb529d80454dda1ae3d7329d150d7aa690 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 4 Mar 2025 16:36:35 +0800 Subject: [PATCH 061/155] =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=88=86=E5=9D=97?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 129 ++++++++++++++++++++++++++++++++++ sdks/storage/filehash.go | 7 ++ 2 files changed, 136 insertions(+) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index e5a8f61..5ceda5f 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -535,3 +535,132 @@ func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectG return nil, jsonResp.ToError() } + +const ObjectNewMultipartUploadPath = "/v1/object/newMultipartUpload" + +type ObjectNewMultipartUpload struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + Path string `json:"path" binding:"required"` +} + +type ObjectNewMultipartUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) { + url, err := url.JoinPath(c.baseURL, ObjectNewMultipartUploadPath) + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(url, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + jsonResp, err := ParseJSONResponse[response[ObjectNewMultipartUploadResp]](resp) + if err != nil { + return nil, err + } + + if jsonResp.Code == errorcode.OK { + return &jsonResp.Data, nil + } + + return nil, jsonResp.ToError() +} + +const ObjectUploadPartPath = "/v1/object/uploadPart" + +type ObjectUploadPart struct { + ObjectUploadPartInfo + File io.ReadCloser `json:"-"` +} + +type ObjectUploadPartInfo struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"` + Index int `json:"index"` +} + +type ObjectUploadPartResp struct{} + +func (c *ObjectService) UploadPart(req ObjectUploadPart) (*ObjectUploadPartResp, error) { + url, err := url.JoinPath(c.baseURL, ObjectUploadPartPath) + if err != nil { + return nil, err + } + + infoJSON, err := serder.ObjectToJSON(req) + if err != nil { + return nil, fmt.Errorf("upload info to json: %w", err) + } + + resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{ + Form: map[string]string{"info": string(infoJSON)}, + Files: iterator.Array(&http2.IterMultiPartFile{ + FieldName: "file", + File: req.File, + }), + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var err error + var codeResp response[ObjectUploadPartResp] + if codeResp, err = serder.JSONToObjectStreamEx[response[ObjectUploadPartResp]](resp.Body); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == errorcode.OK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} + +const ObjectCompleteMultipartUploadPath = "/v1/object/completeMultipartUpload" + +type ObjectCompleteMultipartUpload struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"` + Indexes []int `json:"indexes" binding:"required"` +} + +type ObjectCompleteMultipartUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) { + url, err := url.JoinPath(c.baseURL, ObjectCompleteMultipartUploadPath) + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(url, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + jsonResp, err := ParseJSONResponse[response[ObjectCompleteMultipartUploadResp]](resp) + if err != nil { + return nil, err + } + + if jsonResp.Code == errorcode.OK { + return &jsonResp.Data, nil + } + + return nil, jsonResp.ToError() +} diff --git a/sdks/storage/filehash.go b/sdks/storage/filehash.go index d4f9df5..b60cc30 100644 --- a/sdks/storage/filehash.go +++ b/sdks/storage/filehash.go @@ -20,6 +20,7 @@ type FileHash string const ( FullHashPrefix = "Full" CompositeHashPrefix = "Comp" + EmptyHash = FileHash("Full0000000000000000000000000000000000000000000000000000000000000000") ) func (h *FileHash) GetPrefix() string { @@ -30,6 +31,12 @@ func (h *FileHash) GetHash() string { return string((*h)[4:]) } +// 由调用者保证Hash值有效 +func (h *FileHash) GetHashBytes() []byte { + bytes, _ := hex.DecodeString(h.GetHash()) + return bytes +} + func (h *FileHash) GetHashPrefix(len int) string { return string((*h)[4 : 4+len]) } From e85760ff14c9be62ceb3a86a108fb4c5347d42d8 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 4 Mar 2025 16:52:01 +0800 Subject: [PATCH 062/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 4c4a490..dcb1719 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -56,8 +56,8 @@ const ( ParentType = "parent" PlatformSugon = "sugon" - PlatformOpenI = "openi" - PlatformModelArts = "modelarts" + PlatformOpenI = "OpenI" + PlatformModelArts = "ModelArts" URL = "url" ID = "id" From 94506dcdff42fe663bdb928e2d649d8c02220390 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 5 Mar 2025 14:29:47 +0800 Subject: [PATCH 063/155] =?UTF-8?q?=E8=A7=A3=E5=86=B3BucketPool=E5=85=B3?= =?UTF-8?q?=E9=97=AD=E4=B9=8B=E5=90=8EPool=E5=86=85=E6=9C=89=E4=BA=9B?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=9C=AA=E6=B6=88=E8=B4=B9=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/sync2/bucket_pool.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/sync2/bucket_pool.go b/utils/sync2/bucket_pool.go index f5fa633..d3b4114 100644 --- a/utils/sync2/bucket_pool.go +++ b/utils/sync2/bucket_pool.go @@ -52,12 +52,12 @@ func (p *BucketPool[T]) GetFilled() (T, bool) { p.filledCond.L.Lock() defer p.filledCond.L.Unlock() - if p.closed { - var t T - return t, false - } - if len(p.filled) == 0 { + if p.closed { + var t T + return t, false + } + p.filledCond.Wait() } From 8c629b18b4aef4c75ad1da8792e016286f422424 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 6 Mar 2025 08:58:33 +0800 Subject: [PATCH 064/155] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=8A=B6=E6=80=81=E5=9B=9E=E6=98=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index cbf0ae7..999d8b7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -140,6 +140,7 @@ type CPU struct { serder.Metadata `union:"CPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -147,6 +148,7 @@ type GPU struct { serder.Metadata `union:"GPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -154,6 +156,7 @@ type NPU struct { serder.Metadata `union:"NPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -161,6 +164,7 @@ type Memory struct { serder.Metadata `union:"Memory"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -168,6 +172,7 @@ type DCU struct { serder.Metadata `union:"DCU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -175,6 +180,7 @@ type MLU struct { serder.Metadata `union:"MLU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -182,6 +188,7 @@ type PRICE struct { serder.Metadata `union:"PRICE"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } From 79cb15624e56c36a04dbff287faa76397949c679 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 6 Mar 2025 14:32:06 +0800 Subject: [PATCH 065/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=A2=84=E7=AD=BE?= =?UTF-8?q?=E5=90=8D=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 10 +- go.sum | 15 ++- sdks/storage/cdsapi/presigned.go | 143 ++++++++++++++++++++++++++ sdks/storage/cdsapi/presigned_test.go | 125 ++++++++++++++++++++++ utils/serder/serder.go | 77 ++++++++++++++ utils/serder/serder_test.go | 105 +++++++++++++++++++ 6 files changed, 471 insertions(+), 4 deletions(-) create mode 100644 sdks/storage/cdsapi/presigned.go create mode 100644 sdks/storage/cdsapi/presigned_test.go diff --git a/go.mod b/go.mod index 4e0d2a0..fc87255 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,13 @@ module gitlink.org.cn/cloudream/common -go 1.20 +go 1.22 + +toolchain go1.23.2 require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/credentials v1.17.62 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.15 @@ -21,14 +25,16 @@ require ( golang.org/x/exp v0.0.0-20230519143937-03e91628a987 ) +require github.com/aws/smithy-go v1.22.2 // indirect + require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-querystring v1.1.0 github.com/gopherjs/gopherjs v1.17.2 // indirect - github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect diff --git a/go.sum b/go.sum index c51be4b..f099e44 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ= github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBajbBWGqjNTCa79ocU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -15,15 +21,17 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -51,6 +59,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= +github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -75,6 +84,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -150,6 +160,7 @@ google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go new file mode 100644 index 0000000..ec7f357 --- /dev/null +++ b/sdks/storage/cdsapi/presigned.go @@ -0,0 +1,143 @@ +package cdsapi + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/google/go-querystring/query" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" +) + +const ( + AuthService = "jcs" + AuthRegion = "any" +) + +type PresignedService struct { + *Client + accessKey string + secretKey string +} + +func (c *Client) Presigned(accessKey string, secretKey string) *PresignedService { + return &PresignedService{ + Client: c, + accessKey: accessKey, + secretKey: secretKey, + } +} + +const PresignedObjectDownloadByPathPath = "/v1/presigned/object/downloadByPath" + +type PresignedObjectDownloadByPath struct { + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` + Path string `form:"path" url:"path" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (c *PresignedService) ObjectDownload(req PresignedObjectDownloadByPath, expireIn int) (string, error) { + return c.presign(req, PresignedObjectDownloadByPathPath, http.MethodGet, expireIn) +} + +const PresignedObjectUploadPath = "/v1/presigned/object/upload" + +type PresignedObjectUpload struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID"` + Path string `form:"path" binding:"required" url:"path"` + Affinity cdssdk.StorageID `form:"affinity" url:"affinity,omitempty"` + LoadTo []cdssdk.StorageID `form:"loadTo" url:"loadTo,omitempty"` + LoadToPath []string `form:"loadToPath" url:"loadToPath,omitempty"` +} + +type PresignedObjectUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectUploadPath, http.MethodPost, expireIn) +} + +const PresignedObjectNewMultipartUploadPath = "/v1/presigned/object/newMultipartUpload" + +type PresignedObjectNewMultipartUpload struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID"` + Path string `form:"path" binding:"required" url:"path"` +} + +type PresignedObjectNewMultipartUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultipartUpload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectNewMultipartUploadPath, http.MethodPost, expireIn) +} + +const PresignedObjectUploadPartPath = "/v1/presigned/object/uploadPart" + +type PresignedObjectUploadPart struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + ObjectID cdssdk.ObjectID `form:"objectID" binding:"required" url:"objectID"` + Index int `form:"index" binding:"required" url:"index"` +} + +type PresignedUploadPartResp struct{} + +func (c *PresignedService) ObjectUploadPart(req PresignedObjectUploadPart, expireIn int) (string, error) { + return c.presign(req, PresignedObjectUploadPartPath, http.MethodPost, expireIn) +} + +const PresignedObjectCompleteMultipartUploadPath = "/v1/presigned/object/completeMultipartUpload" + +type PresignedObjectCompleteMultipartUpload struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + ObjectID cdssdk.ObjectID `form:"objectID" binding:"required" url:"objectID"` + Indexes []int `form:"indexes" binding:"required" url:"indexes"` +} + +type PresignedObjectCompleteMultipartUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectCompleteMultipartUpload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectCompleteMultipartUploadPath, http.MethodPost, expireIn) +} + +func (c *PresignedService) presign(req any, path string, method string, expireIn int) (string, error) { + u, err := url.Parse(c.baseURL) + if err != nil { + return "", err + } + u = u.JoinPath(path) + + us, err := query.Values(req) + if err != nil { + return "", err + } + us.Add("X-Expires", fmt.Sprintf("%v", expireIn)) + + u.RawQuery = us.Encode() + + prod := credentials.NewStaticCredentialsProvider(c.accessKey, c.secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return "", err + } + + r, err := http.NewRequest(method, u.String(), nil) + if err != nil { + return "", err + } + + signer := v4.NewSigner() + signedURL, _, err := signer.PresignHTTP(context.Background(), cred, r, "", AuthService, AuthRegion, time.Now()) + return signedURL, err +} diff --git a/sdks/storage/cdsapi/presigned_test.go b/sdks/storage/cdsapi/presigned_test.go new file mode 100644 index 0000000..ae34e3d --- /dev/null +++ b/sdks/storage/cdsapi/presigned_test.go @@ -0,0 +1,125 @@ +package cdsapi + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + "gitlink.org.cn/cloudream/common/pkgs/types" +) + +func Test_Presigned(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned("123456", "123456") + url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ + UserID: 1, + PackageID: 3, + Path: "example.java", + Offset: 1, + Length: types.Ref(int64(100)), + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) + + Convey("上传文件", t, func() { + pre := cli.Presigned("123456", "123456") + url, err := pre.ObjectUpload(PresignedObjectUpload{ + UserID: 1, + PackageID: 3, + Path: "example.java", + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedObjectDownload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned("123456", "123456") + url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ + UserID: 1, + PackageID: 3, + Path: "example.java", + // Offset: 1, + // Length: types.Ref(int64(100)), + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedObjectUpload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + }) + + Convey("上传文件", t, func() { + pre := cli.Presigned("123456", "123456") + url, err := pre.ObjectUpload(PresignedObjectUpload{ + UserID: 1, + PackageID: 3, + Path: "example.java", + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedNewMultipartUpload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + }) + + Convey("启动分片上传", t, func() { + pre := cli.Presigned("123456", "123456") + url, err := pre.ObjectNewMultipartUpload(PresignedObjectNewMultipartUpload{ + UserID: 1, + PackageID: 3, + Path: "example.java", + }, 600) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedObjectUploadPart(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + }) + + Convey("上传分片", t, func() { + pre := cli.Presigned("123456", "123456") + url, err := pre.ObjectUploadPart(PresignedObjectUploadPart{ + UserID: 1, + ObjectID: 7, + Index: 3, + }, 600) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedCompleteMultipartUpload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + }) + + Convey("合并分片", t, func() { + pre := cli.Presigned("123456", "123456") + url, err := pre.ObjectCompleteMultipartUpload(PresignedObjectCompleteMultipartUpload{ + UserID: 1, + ObjectID: 7, + Indexes: []int{1, 2, 3}, + }, 600) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} diff --git a/utils/serder/serder.go b/utils/serder/serder.go index c665201..9967ae5 100644 --- a/utils/serder/serder.go +++ b/utils/serder/serder.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "reflect" + "strings" jsoniter "github.com/json-iterator/go" "github.com/mitchellh/mapstructure" @@ -189,3 +190,79 @@ func ObjectToMap(obj any) (map[string]any, error) { } return mp, dec.Decode(obj) } + +// 1. 尝试解开所有引用 +// +// 2. nil值将会是空字符串 +func ObjectToMapString(obj any) (map[string]string, error) { + if obj == nil { + return make(map[string]string), nil + } + + v := reflect.ValueOf(obj) + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + if !v.IsValid() { + return make(map[string]string), nil + } + if v.Kind() != reflect.Struct { + return nil, fmt.Errorf("type %v is not a struct", v.Type()) + } + + mp := make(map[string]string) + objectToMapString(v, mp) + return mp, nil +} + +func objectToMapString(val reflect.Value, mp map[string]string) { + typ := val.Type() + for i := 0; i < val.NumField(); i++ { + vf := val.Field(i) + tf := typ.Field(i) + if tf.Anonymous { + objectToMapString(vf, mp) + continue + } + + fieldName := tf.Name + + omitEmpty := false + jsonTag := tf.Tag.Get("json") + if jsonTag != "" { + tagParts := strings.Split(jsonTag, ",") + fieldName = strings.TrimSpace(tagParts[0]) + if len(tagParts) > 1 { + for _, tagPart := range tagParts[1:] { + tagPart = strings.TrimSpace(tagPart) + if tagPart == "omitempty" { + omitEmpty = true + } + } + } + } + + for vf.Kind() == reflect.Ptr { + vf = vf.Elem() + } + + if !vf.IsValid() { + if omitEmpty { + continue + } + + mp[fieldName] = "" + continue + } + + if vf.IsZero() && omitEmpty { + continue + } + + if vf.Kind() == reflect.Array { + + } + + mp[fieldName] = fmt.Sprintf("%v", vf) + } +} diff --git a/utils/serder/serder_test.go b/utils/serder/serder_test.go index abd7aef..e0d1f4c 100644 --- a/utils/serder/serder_test.go +++ b/utils/serder/serder_test.go @@ -694,3 +694,108 @@ func Test_ObjectToJSONEx4(t *testing.T) { So(ret[0].(*StCallback).Value, ShouldEqual, "called") }) } + +type StStringer struct { +} + +func (s StStringer) String() string { + return "StStringer" +} + +func Test_ObjectToMapString(t *testing.T) { + Convey("结构体", t, func() { + type StEmb struct { + IntRef *int `json:"intRef,omitempty"` + BoolRef **bool `json:"boolRef"` + StrRef *string `json:"strRef"` + } + + type St struct { + StEmb + Int int + Bool bool + Str string + StStringer *StStringer + } + + st := St{ + StEmb: StEmb{ + IntRef: types.Ref(123), + BoolRef: types.Ref(types.Ref(true)), + }, + Int: 456, + Bool: false, + Str: "test", + StStringer: &StStringer{}, + } + + mp, err := ObjectToMapString(st) + So(err, ShouldBeNil) + + So(mp["intRef"], ShouldEqual, "123") + So(mp["boolRef"], ShouldEqual, "true") + So(mp["strRef"], ShouldEqual, "") + So(mp["Int"], ShouldEqual, "456") + So(mp["Bool"], ShouldEqual, "false") + So(mp["Str"], ShouldEqual, "test") + So(mp["StStringer"], ShouldEqual, "StStringer") + }) + + Convey("结构体引用", t, func() { + type StEmb struct { + IntRef *int `json:"intRef,omitempty"` + BoolRef **bool `json:"boolRef"` + StrRef *string `json:"strRef"` + } + + type St struct { + StEmb + Int int + Bool bool + Str string + StStringer *StStringer + } + + st := St{ + StEmb: StEmb{ + IntRef: types.Ref(123), + BoolRef: types.Ref(types.Ref(true)), + }, + Int: 456, + Bool: false, + Str: "test", + StStringer: &StStringer{}, + } + + mp, err := ObjectToMapString(&st) + So(err, ShouldBeNil) + + So(mp["intRef"], ShouldEqual, "123") + So(mp["boolRef"], ShouldEqual, "true") + So(mp["strRef"], ShouldEqual, "") + So(mp["Int"], ShouldEqual, "456") + So(mp["Bool"], ShouldEqual, "false") + So(mp["Str"], ShouldEqual, "test") + So(mp["StStringer"], ShouldEqual, "StStringer") + }) + + Convey("nil", t, func() { + mp, err := ObjectToMapString(nil) + So(err, ShouldBeNil) + So(mp, ShouldResemble, map[string]string{}) + }) + + Convey("nil指针", t, func() { + type St struct{} + var st *St + mp, err := ObjectToMapString(st) + So(err, ShouldBeNil) + So(mp, ShouldResemble, map[string]string{}) + }) + + Convey("非结构体", t, func() { + mp, err := ObjectToMapString(123) + So(err, ShouldNotBeNil) + So(mp, ShouldBeNil) + }) +} From dad73eba53d24a7d8fc2ff98ed7a51b0a32b8d0c Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 7 Mar 2025 15:03:38 +0800 Subject: [PATCH 066/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 2 ++ sdks/pcmscheduler/models.go | 11 ++++++++++- sdks/scheduler/models.go | 1 + sdks/uploader/models.go | 12 ++++++------ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 6f6a366..269151a 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -163,6 +163,8 @@ func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, erro type RunJobReq struct { TaskID TaskID `json:"taskID"` + JobSetID schsdk.JobSetID `json:"jobSetID"` + LocalJobID string `json:"localJobID"` ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index dcb1719..daca965 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -24,6 +24,7 @@ const ( DATASET = "dataset" IMAGE = "image" MODEL = "model" + RESULT = "result" OrderByName = "name" OrderBySize = "size" @@ -278,10 +279,18 @@ type TaskMessage struct { Message string `json:"message"` } +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` +} + type UploadParams struct { DataType string `json:"dataType"` UploadInfo UploadInfo `json:"uploadInfo"` - //UploadPriority UploadPriority `json:"uploadPriority"` } type UploadInfo interface { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 999d8b7..6ba148c 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -220,6 +220,7 @@ type DataReturnJobInfo struct { JobInfoBase Type string `json:"type"` BucketID cdssdk.BucketID `json:"bucketID"` + BindingType string `json:"bindingType"` TargetLocalJobID string `json:"targetLocalJobID"` } diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 7a9879a..0637b8c 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -112,12 +112,12 @@ func (Cluster) TableName() string { } type Package struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:create_time" json:"createTime"` Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` From 41767cd308f1d7aafad37194ae79253c3cad3ad9 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 10 Mar 2025 15:42:38 +0800 Subject: [PATCH 067/155] =?UTF-8?q?=E5=AD=98=E5=82=A8=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=A2=9E=E5=8A=A0=E9=89=B4=E6=9D=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 54 ++-- go.sum | 133 ++++----- sdks/sdks.go | 183 ++++++++++++- sdks/storage/cdsapi/bucket.go | 136 +++------- sdks/storage/cdsapi/cache.go | 37 +-- sdks/storage/cdsapi/client.go | 4 +- sdks/storage/cdsapi/config.go | 4 +- sdks/storage/cdsapi/hub.go | 38 +-- sdks/storage/cdsapi/hub_io.go | 10 +- sdks/storage/cdsapi/object.go | 371 ++++++++------------------ sdks/storage/cdsapi/package.go | 245 ++++++----------- sdks/storage/cdsapi/presigned.go | 12 +- sdks/storage/cdsapi/presigned_test.go | 18 +- sdks/storage/cdsapi/storage.go | 112 +++----- sdks/storage/cdsapi/user.go | 66 ++--- sdks/storage/cdsapi/utils.go | 68 +++++ utils/io2/io.go | 14 + utils/serder/serder.go | 10 + 18 files changed, 691 insertions(+), 824 deletions(-) diff --git a/go.mod b/go.mod index fc87255..00dc24a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module gitlink.org.cn/cloudream/common -go 1.22 +go 1.23.0 toolchain go1.23.2 @@ -8,7 +8,7 @@ require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/credentials v1.17.62 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.15 github.com/json-iterator/go v1.1.12 @@ -16,41 +16,41 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/modern-go/reflect2 v1.0.2 github.com/otiai10/copy v1.12.0 - github.com/samber/lo v1.36.0 - github.com/sirupsen/logrus v1.9.2 - github.com/smartystreets/goconvey v1.8.0 + github.com/samber/lo v1.49.1 + github.com/sirupsen/logrus v1.9.3 + github.com/smartystreets/goconvey v1.8.1 github.com/streadway/amqp v1.1.0 github.com/zyedidia/generic v1.2.1 - go.etcd.io/etcd/client/v3 v3.5.9 - golang.org/x/exp v0.0.0-20230519143937-03e91628a987 + go.etcd.io/etcd/client/v3 v3.5.12 + golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa ) -require github.com/aws/smithy-go v1.22.2 // indirect +require ( + github.com/aws/smithy-go v1.22.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/smarty/assertions v1.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect +) require ( - github.com/benbjohnson/clock v1.3.0 // indirect - github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/smartystreets/assertions v1.13.1 // indirect - github.com/stretchr/testify v1.8.2 // indirect - go.etcd.io/etcd/api/v3 v3.5.9 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/goleak v1.1.12 // indirect - go.uber.org/multierr v1.9.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect - google.golang.org/grpc v1.54.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + go.etcd.io/etcd/api/v3 v3.5.12 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect ) diff --git a/go.sum b/go.sum index f099e44..58cfdfc 100644 --- a/go.sum +++ b/go.sum @@ -6,30 +6,27 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBa github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8= github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -45,122 +42,96 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw= -github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= -github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= -github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= -github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= +github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM= github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= -github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= -go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= -go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= -go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= -go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= -go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= -go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= +go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= +go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= +go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= +go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA= -golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/sdks/sdks.go b/sdks/sdks.go index c556cd5..1fedfad 100644 --- a/sdks/sdks.go +++ b/sdks/sdks.go @@ -1,6 +1,19 @@ package sdks -import "fmt" +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/google/go-querystring/query" + "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/io2" + "gitlink.org.cn/cloudream/common/utils/serder" +) type CodeMessageError struct { Code string @@ -10,3 +23,171 @@ type CodeMessageError struct { func (e *CodeMessageError) Error() string { return fmt.Sprintf("code: %s, message: %s", e.Code, e.Message) } + +type RequestParam struct { + Method string + Path string + Query url.Values + Header http.Header + Body RequestBody +} + +func (p *RequestParam) MakeRequest(baseURL string) (*http.Request, error) { + var body io.ReadCloser + bodyLen := int64(-1) + if p.Body != nil { + body = p.Body.IntoStream() + bodyLen = p.Body.Length() + } + + req, err := http.NewRequest(p.Method, joinUrlUnsafe(baseURL, p.Path), body) + if err != nil { + return nil, err + } + req.ContentLength = bodyLen + req.URL.RawQuery = p.Query.Encode() + req.Header = p.Header + + return req, nil +} + +func (p *RequestParam) Do(baseURL string, cli *http.Client) (*http.Response, error) { + req, err := p.MakeRequest(baseURL) + if err != nil { + return nil, err + } + + return cli.Do(req) +} + +func joinUrlUnsafe(base string, path string) string { + if strings.HasSuffix(base, "/") { + if strings.HasPrefix(path, "/") { + return base + path[1:] + } + + return base + path + } + + if strings.HasPrefix(path, "/") { + return base + path + } + + return base + "/" + path +} + +type RequestBody interface { + // 请求体长度，如果长度未知，返回-1 + Length() int64 + // 将内部值变成一个流，用于发送请求 + IntoStream() io.ReadCloser +} + +type StringBody struct { + Value string +} + +func (s StringBody) Length() int64 { + return int64(len(s.Value)) +} + +func (s StringBody) IntoStream() io.ReadCloser { + return io.NopCloser(bytes.NewReader([]byte(s.Value))) +} + +type BytesBody struct { + Value []byte +} + +func (b BytesBody) Length() int64 { + return int64(len(b.Value)) +} + +func (b BytesBody) IntoStream() io.ReadCloser { + return io.NopCloser(bytes.NewReader(b.Value)) +} + +type StreamBody struct { + Stream io.ReadCloser + LengthHint int64 // 长度提示，如果长度未知，可以设置为-1 +} + +func (s StreamBody) Length() int64 { + return s.LengthHint +} + +func (s StreamBody) IntoStream() io.ReadCloser { + return s.Stream +} + +type APIRequest interface { + MakeParam() *RequestParam +} + +func MakeJSONParam(method string, path string, body any) *RequestParam { + data, err := serder.ObjectToJSONEx(body) + if err != nil { + // 开发人员应该保证param是可序列化的 + panic(err) + } + + return &RequestParam{ + Method: method, + Path: path, + Body: BytesBody{Value: data}, + } +} + +func MakeQueryParam(method string, path string, q any) *RequestParam { + values, err := query.Values(q) + if err != nil { + // 开发人员应该保证param是可序列化的 + panic(err) + } + + return &RequestParam{ + Method: method, + Path: path, + Query: values, + } +} + +type APIResponse interface { + ParseResponse(resp *http.Response) error +} + +type CodeDataResponse[T any] struct { + Code string `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +func (r *CodeDataResponse[T]) Unwarp() (T, error) { + if r.Code == errorcode.OK { + return r.Data, nil + } + var def T + return def, &CodeMessageError{Code: r.Code, Message: r.Message} +} + +func ParseCodeDataJSONResponse[T any](resp *http.Response, ret T) error { + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var err error + var r CodeDataResponse[T] + + if err = serder.JSONToObjectStreamExRaw(resp.Body, &r); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + return nil + } + + cont, err := io2.ReadMost(resp.Body, 200) + if err != nil { + return fmt.Errorf("unknow response content type: %s, status: %d", contType, resp.StatusCode) + } + strCont := string(cont) + + return fmt.Errorf("unknow response content type: %s, status: %d, body[:200]: %s", contType, resp.StatusCode, strCont) +} diff --git a/sdks/storage/cdsapi/bucket.go b/sdks/storage/cdsapi/bucket.go index 2cb2e9c..0299949 100644 --- a/sdks/storage/cdsapi/bucket.go +++ b/sdks/storage/cdsapi/bucket.go @@ -1,11 +1,10 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) type BucketService struct { @@ -19,36 +18,24 @@ func (c *Client) Bucket() *BucketService { const BucketGetByNamePath = "/bucket/getByName" type BucketGetByName struct { - UserID cdssdk.UserID `json:"userID" form:"userID" binding:"required"` - Name string `json:"name" form:"name" binding:"required"` + UserID cdssdk.UserID `url:"userID" form:"userID" binding:"required"` + Name string `url:"name" form:"name" binding:"required"` } + +func (r *BucketGetByName) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, BucketGetByNamePath, r) +} + type BucketGetByNameResp struct { Bucket cdssdk.Bucket `json:"bucket"` } -func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) { - url, err := url.JoinPath(c.baseURL, BucketGetByNamePath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketGetByNameResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) { + return JSONAPI[*BucketGetByNameResp](c.cfg, http.DefaultClient, &req) } const BucketCreatePath = "/bucket/create" @@ -58,33 +45,20 @@ type BucketCreate struct { Name string `json:"name" binding:"required"` } +func (r *BucketCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, BucketCreatePath, r) +} + type BucketCreateResp struct { Bucket cdssdk.Bucket `json:"bucket"` } -func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) { - url, err := url.JoinPath(c.baseURL, BucketCreatePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketCreateResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *BucketCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) { + return JSONAPI[*BucketCreateResp](c.cfg, http.DefaultClient, &req) } const BucketDeletePath = "/bucket/delete" @@ -94,64 +68,38 @@ type BucketDelete struct { BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` } -type BucketDeleteResp struct{} - -func (c *BucketService) Delete(req BucketDelete) error { - url, err := url.JoinPath(c.baseURL, BucketDeletePath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +func (r *BucketDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, BucketDeletePath, r) +} - codeResp, err := ParseJSONResponse[response[BucketDeleteResp]](resp) - if err != nil { - return err - } +type BucketDeleteResp struct{} - if codeResp.Code == errorcode.OK { - return nil - } +func (r *BucketDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return codeResp.ToError() +func (c *BucketService) Delete(req BucketDelete) error { + return JSONAPINoData[*BucketDeleteResp](c.cfg, http.DefaultClient, &req) } const BucketListUserBucketsPath = "/bucket/listUserBuckets" type BucketListUserBucketsReq struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` +} + +func (r *BucketListUserBucketsReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, BucketListUserBucketsPath, r) } type BucketListUserBucketsResp struct { Buckets []cdssdk.Bucket `json:"buckets"` } +func (r *BucketListUserBucketsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *BucketService) ListUserBuckets(req BucketListUserBucketsReq) (*BucketListUserBucketsResp, error) { - url, err := url.JoinPath(c.baseURL, BucketListUserBucketsPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketListUserBucketsResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() + return JSONAPI[*BucketListUserBucketsResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/cache.go b/sdks/storage/cdsapi/cache.go index 609cdd0..bb28e58 100644 --- a/sdks/storage/cdsapi/cache.go +++ b/sdks/storage/cdsapi/cache.go @@ -1,11 +1,10 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) const CacheMovePackagePath = "/cache/movePackage" @@ -15,29 +14,17 @@ type CacheMovePackageReq struct { PackageID cdssdk.PackageID `json:"packageID"` StorageID cdssdk.StorageID `json:"storageID"` } + +func (r *CacheMovePackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, CacheMovePackagePath, r) +} + type CacheMovePackageResp struct{} +func (r *CacheMovePackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) CacheMovePackage(req CacheMovePackageReq) (*CacheMovePackageResp, error) { - url, err := url.JoinPath(c.baseURL, CacheMovePackagePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[CacheMovePackageResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } - - return nil, jsonResp.ToError() + return JSONAPI[*CacheMovePackageResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/client.go b/sdks/storage/cdsapi/client.go index 40af4dd..8e12be3 100644 --- a/sdks/storage/cdsapi/client.go +++ b/sdks/storage/cdsapi/client.go @@ -18,12 +18,12 @@ func (r *response[T]) ToError() *sdks.CodeMessageError { } type Client struct { - baseURL string + cfg *Config } func NewClient(cfg *Config) *Client { return &Client{ - baseURL: cfg.URL, + cfg: cfg, } } diff --git a/sdks/storage/cdsapi/config.go b/sdks/storage/cdsapi/config.go index 484808c..448039e 100644 --- a/sdks/storage/cdsapi/config.go +++ b/sdks/storage/cdsapi/config.go @@ -1,5 +1,7 @@ package cdsapi type Config struct { - URL string `json:"url"` + URL string `json:"url"` + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` } diff --git a/sdks/storage/cdsapi/hub.go b/sdks/storage/cdsapi/hub.go index d1ed7b7..fb360a4 100644 --- a/sdks/storage/cdsapi/hub.go +++ b/sdks/storage/cdsapi/hub.go @@ -1,44 +1,30 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) var HubGetHubsPath = "/hub/getHubs" type HubGetHubsReq struct { - HubIDs []cdssdk.HubID `json:"hubIDs"` + HubIDs []cdssdk.HubID `form:"hubIDs" url:"hubIDs"` +} + +func (r *HubGetHubsReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, HubGetHubsPath, r) } type HubGetHubsResp struct { Hubs []*cdssdk.Hub `json:"hubs"` } +func (r *HubGetHubsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) HubGetHubs(req HubGetHubsReq) (*HubGetHubsResp, error) { - url, err := url.JoinPath(c.baseURL, HubGetHubsPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[HubGetHubsResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } - - return nil, jsonResp.ToError() + return JSONAPI[*HubGetHubsResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/hub_io.go b/sdks/storage/cdsapi/hub_io.go index bd5f1a3..826c425 100644 --- a/sdks/storage/cdsapi/hub_io.go +++ b/sdks/storage/cdsapi/hub_io.go @@ -24,7 +24,7 @@ type GetStreamReq struct { } func (c *Client) GetStream(req GetStreamReq) (io.ReadCloser, error) { - targetUrl, err := url.JoinPath(c.baseURL, GetStreamPath) + targetUrl, err := url.JoinPath(c.cfg.URL, GetStreamPath) if err != nil { return nil, err } @@ -65,7 +65,7 @@ type SendStreamInfo struct { } func (c *Client) SendStream(req SendStreamReq) error { - targetUrl, err := url.JoinPath(c.baseURL, SendStreamPath) + targetUrl, err := url.JoinPath(c.cfg.URL, SendStreamPath) if err != nil { return err } @@ -133,7 +133,7 @@ type ExecuteIOPlanReq struct { } func (c *Client) ExecuteIOPlan(req ExecuteIOPlanReq) error { - targetUrl, err := url.JoinPath(c.baseURL, ExecuteIOPlanPath) + targetUrl, err := url.JoinPath(c.cfg.URL, ExecuteIOPlanPath) if err != nil { return err } @@ -171,7 +171,7 @@ type SendVarReq struct { } func (c *Client) SendVar(req SendVarReq) error { - targetUrl, err := url.JoinPath(c.baseURL, SendVarPath) + targetUrl, err := url.JoinPath(c.cfg.URL, SendVarPath) if err != nil { return err } @@ -214,7 +214,7 @@ type GetVarResp struct { } func (c *Client) GetVar(req GetVarReq) (*GetVarResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, GetVarPath) + targetUrl, err := url.JoinPath(c.cfg.URL, GetVarPath) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 5ceda5f..80fc18f 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -4,12 +4,14 @@ import ( "fmt" "io" "mime" + "net/http" "net/url" "strings" "time" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -28,76 +30,52 @@ func (c *Client) Object() *ObjectService { const ObjectListPathByPath = "/object/listByPath" type ObjectListByPath struct { - UserID cdssdk.UserID `form:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" binding:"required"` - Path string `form:"path"` // 允许为空字符串 - IsPrefix bool `form:"isPrefix"` - NoRecursive bool `form:"noRecursive"` // 仅当isPrefix为true时有效，表示仅查询直接属于Prefix下的对象，对于更深的对象，返回它们的公共前缀 + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID"` + Path string `form:"path" url:"path"` // 允许为空字符串 + IsPrefix bool `form:"isPrefix" url:"isPrefix"` + NoRecursive bool `form:"noRecursive" url:"noRecursive"` // 仅当isPrefix为true时有效，表示仅查询直接属于Prefix下的对象，对于更深的对象，返回它们的公共前缀 } + +func (r *ObjectListByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectListPathByPath, r) +} + type ObjectListByPathResp struct { CommonPrefixes []string `json:"commonPrefixes"` // 仅在IsPrefix为true且NoRecursive为true时有效，包含更深层对象的shared prefix Objects []cdssdk.Object `json:"objects"` // 如果IsPrefix为true且NoRecursive为false，则返回所有匹配的对象，否则只返回直接属于Prefix下的对象 } -func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectListPathByPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectListByPathResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectListByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { + return JSONAPI[*ObjectListByPathResp](c.cfg, http.DefaultClient, &req) } const ObjectListByIDsPath = "/object/listByIDs" type ObjectListByIDs struct { - UserID cdssdk.UserID `json:"userID" binding:"required"` - ObjectIDs []cdssdk.ObjectID `json:"objectIDs" binding:"required"` + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + ObjectIDs []cdssdk.ObjectID `form:"objectIDs" binding:"required" url:"objectIDs"` +} + +func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) } + type ObjectListByIDsResp struct { Objects []*cdssdk.Object `json:"object"` // 与ObjectIDs一一对应，如果某个ID不存在，则对应位置为nil } -func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectListByIDsPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectListByIDsResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectListByIDsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) { + return JSONAPI[*ObjectListByIDsResp](c.cfg, http.DefaultClient, &req) } const ObjectUploadPath = "/object/upload" @@ -127,7 +105,7 @@ type ObjectUploadResp struct { } func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUploadPath) + url, err := url.JoinPath(c.cfg.URL, ObjectUploadPath) if err != nil { return nil, err } @@ -184,7 +162,7 @@ type DownloadingObject struct { } func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) { - url, err := url.JoinPath(c.baseURL, ObjectDownloadPath) + url, err := url.JoinPath(c.cfg.URL, ObjectDownloadPath) if err != nil { return nil, err } @@ -229,7 +207,7 @@ type ObjectDownloadByPath struct { } func (c *ObjectService) DownloadByPath(req ObjectDownloadByPath) (*DownloadingObject, error) { - url, err := url.JoinPath(c.baseURL, ObjectDownloadByPathPath) + url, err := url.JoinPath(c.cfg.URL, ObjectDownloadByPathPath) if err != nil { return nil, err } @@ -279,33 +257,20 @@ type ObjectUpdateInfo struct { Updatings []UpdatingObject `json:"updatings" binding:"required"` } +func (r *ObjectUpdateInfo) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectUpdateInfoPath, r) +} + type ObjectUpdateInfoResp struct { Successes []cdssdk.ObjectID `json:"successes"` } -func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectUpdateInfoResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) { + return JSONAPI[*ObjectUpdateInfoResp](c.cfg, http.DefaultClient, &req) } const ObjectUpdateInfoByPathPath = "/object/updateInfoByPath" @@ -317,31 +282,18 @@ type ObjectUpdateInfoByPath struct { UpdateTime time.Time `json:"updateTime" binding:"required"` } -type ObjectUpdateInfoByPathResp struct{} - -func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoByPathPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } +func (r *ObjectUpdateInfoByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectUpdateInfoByPathPath, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoByPathResp]](resp) - if err != nil { - return nil, err - } +type ObjectUpdateInfoByPathResp struct{} - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectUpdateInfoByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) { + return JSONAPI[*ObjectUpdateInfoByPathResp](c.cfg, http.DefaultClient, &req) } const ObjectMovePath = "/object/move" @@ -362,33 +314,20 @@ type ObjectMove struct { Movings []MovingObject `json:"movings" binding:"required"` } +func (r *ObjectMove) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectMovePath, r) +} + type ObjectMoveResp struct { Successes []cdssdk.ObjectID `json:"successes"` } -func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectMovePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectMoveResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectMoveResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) { + return JSONAPI[*ObjectMoveResp](c.cfg, http.DefaultClient, &req) } const ObjectDeletePath = "/object/delete" @@ -398,31 +337,18 @@ type ObjectDelete struct { ObjectIDs []cdssdk.ObjectID `json:"objectIDs" binding:"required"` } -type ObjectDeleteResp struct{} - -func (c *ObjectService) Delete(req ObjectDelete) error { - url, err := url.JoinPath(c.baseURL, ObjectDeletePath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +func (r *ObjectDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectDeletePath, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectDeleteResp]](resp) - if err != nil { - return err - } +type ObjectDeleteResp struct{} - if jsonResp.Code == errorcode.OK { - return nil - } +func (r *ObjectDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return jsonResp.ToError() +func (c *ObjectService) Delete(req ObjectDelete) error { + return JSONAPINoData[*ObjectDeleteResp](c.cfg, http.DefaultClient, &req) } const ObjectDeleteByPathPath = "/object/deleteByPath" @@ -432,31 +358,19 @@ type ObjectDeleteByPath struct { PackageID cdssdk.PackageID `json:"packageID" binding:"required"` Path string `json:"path" binding:"required"` } -type ObjectDeleteByPathResp struct{} -func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error { - url, err := url.JoinPath(c.baseURL, ObjectDeleteByPathPath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +func (r *ObjectDeleteByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectDeleteByPathPath, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectDeleteByPathResp]](resp) - if err != nil { - return err - } +type ObjectDeleteByPathResp struct{} - if jsonResp.Code == errorcode.OK { - return nil - } +func (r *ObjectDeleteByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return jsonResp.ToError() +func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error { + return JSONAPINoData[*ObjectDeleteByPathResp](c.cfg, http.DefaultClient, &req) } const ObjectClonePath = "/object/clone" @@ -466,6 +380,10 @@ type ObjectClone struct { Clonings []CloningObject `json:"clonings" binding:"required"` } +func (r *ObjectClone) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectClonePath, r) +} + type CloningObject struct { ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"` NewPath string `json:"newPath" binding:"required"` @@ -476,64 +394,35 @@ type ObjectCloneResp struct { Objects []*cdssdk.Object `json:"objects"` } -func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectClonePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectCloneResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectCloneResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) { + return JSONAPI[*ObjectCloneResp](c.cfg, http.DefaultClient, &req) } const ObjectGetPackageObjectsPath = "/object/getPackageObjects" type ObjectGetPackageObjects struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` } + +func (r *ObjectGetPackageObjects) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectGetPackageObjectsPath, r) +} + type ObjectGetPackageObjectsResp struct { Objects []cdssdk.Object `json:"objects"` } -func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectGetPackageObjectsPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectGetPackageObjectsResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectGetPackageObjectsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) { + return JSONAPI[*ObjectGetPackageObjectsResp](c.cfg, http.DefaultClient, &req) } const ObjectNewMultipartUploadPath = "/v1/object/newMultipartUpload" @@ -544,33 +433,20 @@ type ObjectNewMultipartUpload struct { Path string `json:"path" binding:"required"` } +func (r *ObjectNewMultipartUpload) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectNewMultipartUploadPath, r) +} + type ObjectNewMultipartUploadResp struct { Object cdssdk.Object `json:"object"` } -func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectNewMultipartUploadPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectNewMultipartUploadResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectNewMultipartUploadResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) { + return JSONAPI[*ObjectNewMultipartUploadResp](c.cfg, http.DefaultClient, &req) } const ObjectUploadPartPath = "/v1/object/uploadPart" @@ -589,7 +465,7 @@ type ObjectUploadPartInfo struct { type ObjectUploadPartResp struct{} func (c *ObjectService) UploadPart(req ObjectUploadPart) (*ObjectUploadPartResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUploadPartPath) + url, err := url.JoinPath(c.cfg.URL, ObjectUploadPartPath) if err != nil { return nil, err } @@ -636,31 +512,18 @@ type ObjectCompleteMultipartUpload struct { Indexes []int `json:"indexes" binding:"required"` } +func (r *ObjectCompleteMultipartUpload) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectCompleteMultipartUploadPath, r) +} + type ObjectCompleteMultipartUploadResp struct { Object cdssdk.Object `json:"object"` } -func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectCompleteMultipartUploadPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectCompleteMultipartUploadResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectCompleteMultipartUploadResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) { + return JSONAPI[*ObjectCompleteMultipartUploadResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index e72ae3e..085871f 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -2,11 +2,12 @@ package cdsapi import ( "fmt" + "net/http" "net/url" - "strings" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -23,72 +24,48 @@ func (c *Client) Package() *PackageService { const PackageGetPath = "/package/get" type PackageGetReq struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` } + +func (r *PackageGetReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetPath, r) +} + type PackageGetResp struct { cdssdk.Package } -func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageGetResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { + return JSONAPI[*PackageGetResp](c.cfg, http.DefaultClient, &req) } const PackageGetByFullNamePath = "/package/getByFullName" type PackageGetByFullName struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - BucketName string `form:"bucketName" json:"bucketName" binding:"required"` - PackageName string `form:"packageName" json:"packageName" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + BucketName string `form:"bucketName" url:"bucketName" binding:"required"` + PackageName string `form:"packageName" url:"packageName" binding:"required"` } + +func (r *PackageGetByFullName) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetByFullNamePath, r) +} + type PackageGetByFullNameResp struct { Package cdssdk.Package `json:"package"` } -func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetByFullNamePath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetByFullNameResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { + return JSONAPI[*PackageGetByFullNameResp](c.cfg, http.DefaultClient, &req) } const PackageCreatePath = "/package/create" @@ -99,33 +76,20 @@ type PackageCreate struct { Name string `json:"name"` } +func (r *PackageCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageCreatePath, r) +} + type PackageCreateResp struct { Package cdssdk.Package `json:"package"` } -func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) { - url, err := url.JoinPath(s.baseURL, PackageCreatePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageCreateResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) { + return JSONAPI[*PackageCreateResp](s.cfg, http.DefaultClient, &req) } const PackageCreateLoadPath = "/package/createLoad" @@ -147,7 +111,7 @@ type PackageCreateLoadResp struct { } func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadResp, error) { - url, err := url.JoinPath(c.baseURL, PackageCreateLoadPath) + url, err := url.JoinPath(c.cfg.URL, PackageCreateLoadPath) if err != nil { return nil, err } @@ -190,35 +154,18 @@ type PackageDelete struct { PackageID cdssdk.PackageID `json:"packageID" binding:"required"` } -func (c *PackageService) Delete(req PackageDelete) error { - url, err := url.JoinPath(c.baseURL, PackageDeletePath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } - - contType := resp.Header.Get("Content-Type") - - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[any] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return fmt.Errorf("parsing response: %w", err) - } +func (r *PackageDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageDeletePath, r) +} - if codeResp.Code == errorcode.OK { - return nil - } +type PackageDeleteResp struct{} - return codeResp.ToError() - } +func (r *PackageDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return fmt.Errorf("unknow response content type: %s", contType) +func (c *PackageService) Delete(req PackageDelete) error { + return JSONAPINoData[*PackageDeleteResp](c.cfg, http.DefaultClient, &req) } const PackageClonePath = "/package/clone" @@ -230,102 +177,64 @@ type PackageClone struct { Name string `json:"name" binding:"required"` } +func (r *PackageClone) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageClonePath, r) +} + type PackageCloneResp struct { Package cdssdk.Package `json:"package"` } -func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) { - url, err := url.JoinPath(c.baseURL, PackageClonePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageCloneResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageCloneResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) { + return JSONAPI[*PackageCloneResp](c.cfg, http.DefaultClient, &req) } const PackageListBucketPackagesPath = "/package/listBucketPackages" type PackageListBucketPackages struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - BucketID cdssdk.BucketID `form:"bucketID" json:"bucketID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + BucketID cdssdk.BucketID `form:"bucketID" url:"bucketID" binding:"required"` +} + +func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageListBucketPackagesPath, r) } type PackageListBucketPackagesResp struct { Packages []cdssdk.Package `json:"packages"` } -func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { - url, err := url.JoinPath(c.baseURL, PackageListBucketPackagesPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageListBucketPackagesResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { + return JSONAPI[*PackageListBucketPackagesResp](c.cfg, http.DefaultClient, &req) } const PackageGetCachedStoragesPath = "/package/getCachedStorages" type PackageGetCachedStoragesReq struct { - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` +} + +func (r *PackageGetCachedStoragesReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetCachedStoragesPath, r) } type PackageGetCachedStoragesResp struct { cdssdk.PackageCachingInfo } -func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetCachedStoragesPath) - if err != nil { - return nil, err - } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetCachedStoragesResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageGetCachedStoragesResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { + return JSONAPI[*PackageGetCachedStoragesResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go index ec7f357..ae83642 100644 --- a/sdks/storage/cdsapi/presigned.go +++ b/sdks/storage/cdsapi/presigned.go @@ -20,15 +20,11 @@ const ( type PresignedService struct { *Client - accessKey string - secretKey string } -func (c *Client) Presigned(accessKey string, secretKey string) *PresignedService { +func (c *Client) Presigned() *PresignedService { return &PresignedService{ - Client: c, - accessKey: accessKey, - secretKey: secretKey, + Client: c, } } @@ -112,7 +108,7 @@ func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectComp } func (c *PresignedService) presign(req any, path string, method string, expireIn int) (string, error) { - u, err := url.Parse(c.baseURL) + u, err := url.Parse(c.cfg.URL) if err != nil { return "", err } @@ -126,7 +122,7 @@ func (c *PresignedService) presign(req any, path string, method string, expireIn u.RawQuery = us.Encode() - prod := credentials.NewStaticCredentialsProvider(c.accessKey, c.secretKey, "") + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") cred, err := prod.Retrieve(context.TODO()) if err != nil { return "", err diff --git a/sdks/storage/cdsapi/presigned_test.go b/sdks/storage/cdsapi/presigned_test.go index ae34e3d..37c5b87 100644 --- a/sdks/storage/cdsapi/presigned_test.go +++ b/sdks/storage/cdsapi/presigned_test.go @@ -9,11 +9,13 @@ import ( func Test_Presigned(t *testing.T) { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", }) Convey("下载文件", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ UserID: 1, PackageID: 3, @@ -26,7 +28,7 @@ func Test_Presigned(t *testing.T) { }) Convey("上传文件", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectUpload(PresignedObjectUpload{ UserID: 1, PackageID: 3, @@ -43,7 +45,7 @@ func Test_PresignedObjectDownload(t *testing.T) { }) Convey("下载文件", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ UserID: 1, PackageID: 3, @@ -62,7 +64,7 @@ func Test_PresignedObjectUpload(t *testing.T) { }) Convey("上传文件", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectUpload(PresignedObjectUpload{ UserID: 1, PackageID: 3, @@ -79,7 +81,7 @@ func Test_PresignedNewMultipartUpload(t *testing.T) { }) Convey("启动分片上传", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectNewMultipartUpload(PresignedObjectNewMultipartUpload{ UserID: 1, PackageID: 3, @@ -96,7 +98,7 @@ func Test_PresignedObjectUploadPart(t *testing.T) { }) Convey("上传分片", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectUploadPart(PresignedObjectUploadPart{ UserID: 1, ObjectID: 7, @@ -113,7 +115,7 @@ func Test_PresignedCompleteMultipartUpload(t *testing.T) { }) Convey("合并分片", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectCompleteMultipartUpload(PresignedObjectCompleteMultipartUpload{ UserID: 1, ObjectID: 7, diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index f45b234..b1986b8 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -1,14 +1,10 @@ package cdsapi import ( - "fmt" - "net/url" - "strings" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" - "gitlink.org.cn/cloudream/common/utils/serder" ) const StorageLoadPackagePath = "/storage/loadPackage" @@ -19,31 +15,19 @@ type StorageLoadPackageReq struct { StorageID cdssdk.StorageID `json:"storageID" binding:"required"` RootPath string `json:"rootPath"` } + +func (r *StorageLoadPackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, StorageLoadPackagePath, r) +} + type StorageLoadPackageResp struct{} +func (r *StorageLoadPackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPackageResp, error) { - url, err := url.JoinPath(c.baseURL, StorageLoadPackagePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[StorageLoadPackageResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() + return JSONAPI[*StorageLoadPackageResp](c.cfg, http.DefaultClient, &req) } const StorageCreatePackagePath = "/storage/createPackage" @@ -57,71 +41,41 @@ type StorageCreatePackageReq struct { StorageAffinity cdssdk.StorageID `json:"storageAffinity"` } +func (r *StorageCreatePackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, StorageCreatePackagePath, r) +} + type StorageCreatePackageResp struct { PackageID cdssdk.PackageID `json:"packageID"` } +func (r *StorageCreatePackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) StorageCreatePackage(req StorageCreatePackageReq) (*StorageCreatePackageResp, error) { - url, err := url.JoinPath(c.baseURL, StorageCreatePackagePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - contType := resp.Header.Get("Content-Type") - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[StorageCreatePackageResp] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return nil, fmt.Errorf("parsing response: %w", err) - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() - } - - return nil, fmt.Errorf("unknow response content type: %s", contType) + return JSONAPI[*StorageCreatePackageResp](c.cfg, http.DefaultClient, &req) } const StorageGetPath = "/storage/get" type StorageGet struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - StorageID cdssdk.StorageID `form:"storageID" json:"storageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + StorageID cdssdk.StorageID `form:"storageID" url:"storageID" binding:"required"` +} + +func (r *StorageGet) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, StorageGetPath, r) } + type StorageGetResp struct { cdssdk.Storage } +func (r *StorageGetResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) StorageGet(req StorageGet) (*StorageGetResp, error) { - url, err := url.JoinPath(c.baseURL, StorageGetPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[StorageGetResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() + return JSONAPI[*StorageGetResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/user.go b/sdks/storage/cdsapi/user.go index 7397f89..295e322 100644 --- a/sdks/storage/cdsapi/user.go +++ b/sdks/storage/cdsapi/user.go @@ -1,11 +1,10 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) const UserCreatePath = "/v1/user/create" @@ -13,32 +12,21 @@ const UserCreatePath = "/v1/user/create" type UserCreate struct { Name string `json:"name" binding:"required"` } + +func (r *UserCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, UserCreatePath, r) +} + type UserCreateResp struct { User cdssdk.User `json:"user"` } -func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { - url, err := url.JoinPath(c.baseURL, UserCreatePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - codeResp, err := ParseJSONResponse[response[UserCreateResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *UserCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { + return JSONAPI[*UserCreateResp](c.cfg, http.DefaultClient, req) } const UserDeletePath = "/v1/user/delete" @@ -47,28 +35,16 @@ type UserDelete struct { UserID cdssdk.UserID `json:"userID" binding:"required"` } -type UserDeleteResp struct{} - -func (c *Client) UserDelete(req *UserDelete) error { - url, err := url.JoinPath(c.baseURL, UserDeletePath) - if err != nil { - return err - } +func (r *UserDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, UserDeletePath, r) +} - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } - codeResp, err := ParseJSONResponse[response[UserDeleteResp]](resp) - if err != nil { - return err - } +type UserDeleteResp struct{} - if codeResp.Code == errorcode.OK { - return nil - } +func (r *UserDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return codeResp.ToError() +func (c *Client) UserDelete(req *UserDelete) error { + return JSONAPINoData[*UserDeleteResp](c.cfg, http.DefaultClient, req) } diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 2ef1bc8..079d50e 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -1,12 +1,17 @@ package cdsapi import ( + "context" "fmt" "io" "net/http" "path/filepath" "strings" + "time" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" + "gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -36,3 +41,66 @@ func ParseJSONResponse[TBody any](resp *http.Response) (TBody, error) { return ret, fmt.Errorf("unknow response content type: %s, status: %d, body(prefix): %s", contType, resp.StatusCode, strCont[:math2.Min(len(strCont), 200)]) } + +func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) (Resp, error) { + var re Resp + + param := req.MakeParam() + + httpReq, err := param.MakeRequest(cfg.URL) + if err != nil { + return re, err + } + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return re, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return re, err + } + } + + resp, err := cli.Do(httpReq) + if err != nil { + return re, err + } + + err = re.ParseResponse(resp) + return re, err +} + +func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) error { + param := req.MakeParam() + + httpReq, err := param.MakeRequest(cfg.URL) + if err != nil { + return err + } + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + } + + resp, err := cli.Do(httpReq) + if err != nil { + return err + } + + return sdks.ParseCodeDataJSONResponse(resp, any(nil)) +} diff --git a/utils/io2/io.go b/utils/io2/io.go index 8a0d113..5107ae0 100644 --- a/utils/io2/io.go +++ b/utils/io2/io.go @@ -196,3 +196,17 @@ func DropWithBuf(str io.Reader, buf []byte) { } } } + +func ReadMost(str io.Reader, n int) ([]byte, error) { + buf := make([]byte, n) + n, err := io.ReadFull(str, buf) + if err == nil { + return buf, nil + } + + if err == io.EOF || err == io.ErrUnexpectedEOF { + return buf[:n], nil + } + + return buf[:n], err +} diff --git a/utils/serder/serder.go b/utils/serder/serder.go index 9967ae5..cbc7fbb 100644 --- a/utils/serder/serder.go +++ b/utils/serder/serder.go @@ -80,6 +80,16 @@ func JSONToObjectStreamEx[T any](stream io.Reader) (T, error) { return ret, nil } +func JSONToObjectStreamExRaw(stream io.Reader, ret any) error { + dec := defaultAPI.NewDecoder(stream) + err := dec.Decode(ret) + if err != nil { + return err + } + + return nil +} + // 将对象转为JSON字符串。如果需要支持解析TypeUnion类型，则使用"Ex"结尾的同名函数。 // // 注：[]byte会被base64编码，如果要JSON内容要给外部解析，那么应该避免使用[]byte。 From 66da854b0bd75e6dd40945a4dfd1237c3dc9ee8d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 10 Mar 2025 16:40:23 +0800 Subject: [PATCH 068/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/job.go | 16 ++++++++++++++++ sdks/pcmscheduler/jobset.go | 3 +-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 sdks/pcmscheduler/job.go diff --git a/sdks/pcmscheduler/job.go b/sdks/pcmscheduler/job.go new file mode 100644 index 0000000..0470ec8 --- /dev/null +++ b/sdks/pcmscheduler/job.go @@ -0,0 +1,16 @@ +package sch + +import ( + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) + +type PCMJob struct { + ID string `gorm:"column:id" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"` + LocalJobID string `gorm:"column:local_job_id" json:"localJobID"` + Param string `gorm:"column:param" json:"param"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 269151a..e4d7cde 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -92,6 +92,7 @@ type DatasetDistribute struct { type CodeDistribute struct { DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` Clusters []DataDetail `json:"clusters"` } @@ -163,8 +164,6 @@ func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, erro type RunJobReq struct { TaskID TaskID `json:"taskID"` - JobSetID schsdk.JobSetID `json:"jobSetID"` - LocalJobID string `json:"localJobID"` ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` } From 510fe885e4c75bfa06ac59e6d8a7d4522ffd6aff Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 10 Mar 2025 17:15:48 +0800 Subject: [PATCH 069/155] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=92=8C=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E6=8E=A5=E5=8F=A3=E6=94=AF=E6=8C=81=E9=89=B4=E6=9D=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 79 +++++++++++++++------- sdks/storage/cdsapi/package.go | 9 ++- sdks/storage/cdsapi/utils.go | 115 ++++++++++++++++++++++++++++++++- 3 files changed, 174 insertions(+), 29 deletions(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 80fc18f..9b4b8ed 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -1,6 +1,7 @@ package cdsapi import ( + "context" "fmt" "io" "mime" @@ -9,6 +10,8 @@ import ( "strings" "time" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/sdks" @@ -115,16 +118,15 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { return nil, fmt.Errorf("upload info to json: %w", err) } - resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{ - Form: map[string]string{"info": string(infoJSON)}, - Files: iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { + resp, err := PostMultiPart(c.cfg, url, + map[string]string{"info": string(infoJSON)}, + iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { return &http2.IterMultiPartFile{ FieldName: "files", FileName: src.Path, File: src.File, }, nil - }), - }) + })) if err != nil { return nil, err } @@ -151,25 +153,42 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { const ObjectDownloadPath = "/object/download" type ObjectDownload struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - ObjectID cdssdk.ObjectID `form:"objectID" json:"objectID" binding:"required"` - Offset int64 `form:"offset" json:"offset,omitempty"` - Length *int64 `form:"length" json:"length,omitempty"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `form:"objectID" url:"objectID" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (r *ObjectDownload) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectDownloadPath, r) } + type DownloadingObject struct { Path string File io.ReadCloser } func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) { - url, err := url.JoinPath(c.cfg.URL, ObjectDownloadPath) + httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL) if err != nil { return nil, err } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) + if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + resp, err := http.DefaultClient.Do(httpReq) if err != nil { return nil, err } @@ -199,22 +218,38 @@ func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) const ObjectDownloadByPathPath = "/object/downloadByPath" type ObjectDownloadByPath struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` - Path string `form:"path" json:"path" binding:"required"` - Offset int64 `form:"offset" json:"offset,omitempty"` - Length *int64 `form:"length" json:"length,omitempty"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` + Path string `form:"path" url:"path" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (r *ObjectDownloadByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectDownloadByPathPath, r) } func (c *ObjectService) DownloadByPath(req ObjectDownloadByPath) (*DownloadingObject, error) { - url, err := url.JoinPath(c.cfg.URL, ObjectDownloadByPathPath) + httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL) if err != nil { return nil, err } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) + if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + resp, err := http.DefaultClient.Do(httpReq) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index 085871f..621f64b 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -121,16 +121,15 @@ func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadRe return nil, fmt.Errorf("upload info to json: %w", err) } - resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{ - Form: map[string]string{"info": string(infoJSON)}, - Files: iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { + resp, err := PostMultiPart(c.cfg, url, + map[string]string{"info": string(infoJSON)}, + iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { return &http2.IterMultiPartFile{ FieldName: "files", FileName: src.Path, File: src.File, }, nil - }), - }) + })) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 079d50e..5c573ae 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -2,15 +2,21 @@ package cdsapi import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" + "mime/multipart" "net/http" + ul "net/url" "path/filepath" "strings" "time" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/google/go-querystring/query" + "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/math2" @@ -60,7 +66,7 @@ func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http. } signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + err = signer.SignHTTP(context.Background(), cred, httpReq, calcSha256(param.Body), AuthService, AuthRegion, time.Now()) if err != nil { return re, err } @@ -91,7 +97,7 @@ func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli } signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + err = signer.SignHTTP(context.Background(), cred, httpReq, calcSha256(param.Body), AuthService, AuthRegion, time.Now()) if err != nil { return err } @@ -104,3 +110,108 @@ func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli return sdks.ParseCodeDataJSONResponse(resp, any(nil)) } + +func calcSha256(body sdks.RequestBody) string { + hasher := sha256.New() + switch body := body.(type) { + case *sdks.StringBody: + return hex.EncodeToString(hasher.Sum([]byte(body.Value))) + + case *sdks.BytesBody: + return hex.EncodeToString(hasher.Sum(body.Value)) + + default: + return "" + } +} + +func PostMultiPart(cfg *Config, url string, info any, files http2.MultiPartFileIterator) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return nil, err + } + + pr, pw := io.Pipe() + muWriter := multipart.NewWriter(pw) + + req.Header.Set("Content-Type", fmt.Sprintf("%s;boundary=%s", http2.ContentTypeMultiPart, muWriter.Boundary())) + + writeResult := make(chan error, 1) + go func() { + writeResult <- func() error { + defer pw.Close() + defer muWriter.Close() + + if info != nil { + mp, err := query.Values(info) + if err != nil { + return fmt.Errorf("formValues object to map failed, err: %w", err) + } + + for k, v := range mp { + err := muWriter.WriteField(k, v[0]) + if err != nil { + return fmt.Errorf("write form field failed, err: %w", err) + } + } + } + + for { + file, err := files.MoveNext() + if err == iterator.ErrNoMoreItem { + break + } + if err != nil { + return fmt.Errorf("opening file: %w", err) + } + + err = sendFileOnePart(muWriter, file.FieldName, file.FileName, file.File) + file.File.Close() + if err != nil { + return err + } + } + + return nil + }() + }() + + req.Body = pr + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + cli := http.Client{} + resp, err := cli.Do(req) + if err != nil { + return nil, err + } + + writeErr := <-writeResult + if writeErr != nil { + return nil, writeErr + } + + return resp, nil +} + +func sendFileOnePart(muWriter *multipart.Writer, fieldName, fileName string, file io.ReadCloser) error { + w, err := muWriter.CreateFormFile(fieldName, ul.PathEscape(fileName)) + if err != nil { + return fmt.Errorf("create form file failed, err: %w", err) + } + + _, err = io.Copy(w, file) + return err +} From 794e9708ad56364c119ef61788e2d8db7e819742 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 09:43:10 +0800 Subject: [PATCH 070/155] =?UTF-8?q?=E6=8B=86=E5=88=86=E7=AD=BE=E5=90=8D?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/presigned.go | 5 -- sdks/storage/cdsapi/signer.go | 113 +++++++++++++++++++++++++++++++ sdks/storage/cdsapi/utils.go | 37 ++-------- 3 files changed, 120 insertions(+), 35 deletions(-) create mode 100644 sdks/storage/cdsapi/signer.go diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go index ae83642..5e755fb 100644 --- a/sdks/storage/cdsapi/presigned.go +++ b/sdks/storage/cdsapi/presigned.go @@ -13,11 +13,6 @@ import ( cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" ) -const ( - AuthService = "jcs" - AuthRegion = "any" -) - type PresignedService struct { *Client } diff --git a/sdks/storage/cdsapi/signer.go b/sdks/storage/cdsapi/signer.go new file mode 100644 index 0000000..b4f8975 --- /dev/null +++ b/sdks/storage/cdsapi/signer.go @@ -0,0 +1,113 @@ +package cdsapi + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "time" + + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" +) + +const ( + AuthService = "jcs" + AuthRegion = "any" +) + +// 对一个请求进行签名，并将签名信息添加到请求头中。 +// +// 会读取请求体计算sha256哈希值。如果hash值已知，可以使用SignWithPayloadHash方法。 +func Sign(req *http.Request, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + payloadHash := "" + if req.Body != nil { + data, err := io.ReadAll(req.Body) + if err != nil { + return err + } + req.Body.Close() + req.Body = io.NopCloser(bytes.NewReader(data)) + + hasher := sha256.New() + hasher.Write(data) + payloadHash = hex.EncodeToString(hasher.Sum(nil)) + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, payloadHash, AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 对一个请求进行签名，并将签名信息添加到请求头中。 +// +// 不计算请求体的哈希，适合上传文件接口。 +func SignWithoutBody(req *http.Request, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 对一个请求进行签名，签名时使用指定的哈希值作为请求体的哈希值。 +// +// 参数payloadHash必须为sha256哈希值的16进制字符串，全小写。 +func SignWithPayloadHash(req *http.Request, payloadHash string, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, payloadHash, AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 生成一个带签名的URL。 +// +// expiration为签名过期时间，单位为秒。 +// +// 签名时不会包含请求体的哈希值。 +func Presign(req *http.Request, accessKey, secretKey string, expiration int) (string, error) { + query := req.URL.Query() + query.Add("X-Expires", fmt.Sprintf("%v", expiration)) + + req.URL.RawQuery = query.Encode() + + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return "", err + } + + signer := v4.NewSigner() + signedURL, _, err := signer.PresignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + return signedURL, err +} diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 5c573ae..4ba07df 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -1,7 +1,6 @@ package cdsapi import ( - "context" "crypto/sha256" "encoding/hex" "fmt" @@ -11,10 +10,7 @@ import ( ul "net/url" "path/filepath" "strings" - "time" - v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/google/go-querystring/query" "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/sdks" @@ -59,14 +55,7 @@ func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http. } if cfg.AccessKey != "" && cfg.SecretKey != "" { - prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") - cred, err := prod.Retrieve(context.TODO()) - if err != nil { - return re, err - } - - signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, httpReq, calcSha256(param.Body), AuthService, AuthRegion, time.Now()) + err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) if err != nil { return re, err } @@ -90,14 +79,7 @@ func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli } if cfg.AccessKey != "" && cfg.SecretKey != "" { - prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") - cred, err := prod.Retrieve(context.TODO()) - if err != nil { - return err - } - - signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, httpReq, calcSha256(param.Body), AuthService, AuthRegion, time.Now()) + err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) if err != nil { return err } @@ -115,10 +97,12 @@ func calcSha256(body sdks.RequestBody) string { hasher := sha256.New() switch body := body.(type) { case *sdks.StringBody: - return hex.EncodeToString(hasher.Sum([]byte(body.Value))) + hasher.Write([]byte(body.Value)) + return hex.EncodeToString(hasher.Sum(nil)) case *sdks.BytesBody: - return hex.EncodeToString(hasher.Sum(body.Value)) + hasher.Write(body.Value) + return hex.EncodeToString(hasher.Sum(nil)) default: return "" @@ -179,14 +163,7 @@ func PostMultiPart(cfg *Config, url string, info any, files http2.MultiPartFileI req.Body = pr if cfg.AccessKey != "" && cfg.SecretKey != "" { - prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") - cred, err := prod.Retrieve(context.TODO()) - if err != nil { - return nil, err - } - - signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + err = SignWithoutBody(req, cfg.AccessKey, cfg.SecretKey) if err != nil { return nil, err } From 8761088ec2d720992fce77323a2b53c3e02f9efc Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 10:32:04 +0800 Subject: [PATCH 071/155] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/sdks.go | 7 ++++++- sdks/storage/cdsapi/bucket.go | 8 ++++---- sdks/storage/cdsapi/cache.go | 2 +- sdks/storage/cdsapi/hub.go | 2 +- sdks/storage/cdsapi/object.go | 28 ++++++++++++++++------------ sdks/storage/cdsapi/package.go | 14 +++++++------- sdks/storage/cdsapi/storage.go | 6 +++--- sdks/storage/cdsapi/user.go | 4 ++-- sdks/storage/cdsapi/utils.go | 17 ++++++++--------- 9 files changed, 48 insertions(+), 40 deletions(-) diff --git a/sdks/sdks.go b/sdks/sdks.go index 1fedfad..857c6b7 100644 --- a/sdks/sdks.go +++ b/sdks/sdks.go @@ -34,7 +34,7 @@ type RequestParam struct { func (p *RequestParam) MakeRequest(baseURL string) (*http.Request, error) { var body io.ReadCloser - bodyLen := int64(-1) + bodyLen := int64(0) if p.Body != nil { body = p.Body.IntoStream() bodyLen = p.Body.Length() @@ -175,11 +175,16 @@ func ParseCodeDataJSONResponse[T any](resp *http.Response, ret T) error { if strings.Contains(contType, http2.ContentTypeJSON) { var err error var r CodeDataResponse[T] + r.Data = ret if err = serder.JSONToObjectStreamExRaw(resp.Body, &r); err != nil { return fmt.Errorf("parsing response: %w", err) } + if r.Code != errorcode.OK { + return &CodeMessageError{Code: r.Code, Message: r.Message} + } + return nil } diff --git a/sdks/storage/cdsapi/bucket.go b/sdks/storage/cdsapi/bucket.go index 0299949..f0c1ad1 100644 --- a/sdks/storage/cdsapi/bucket.go +++ b/sdks/storage/cdsapi/bucket.go @@ -35,7 +35,7 @@ func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error { } func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) { - return JSONAPI[*BucketGetByNameResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketGetByNameResp{}) } const BucketCreatePath = "/bucket/create" @@ -58,7 +58,7 @@ func (r *BucketCreateResp) ParseResponse(resp *http.Response) error { } func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) { - return JSONAPI[*BucketCreateResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketCreateResp{}) } const BucketDeletePath = "/bucket/delete" @@ -79,7 +79,7 @@ func (r *BucketDeleteResp) ParseResponse(resp *http.Response) error { } func (c *BucketService) Delete(req BucketDelete) error { - return JSONAPINoData[*BucketDeleteResp](c.cfg, http.DefaultClient, &req) + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const BucketListUserBucketsPath = "/bucket/listUserBuckets" @@ -101,5 +101,5 @@ func (r *BucketListUserBucketsResp) ParseResponse(resp *http.Response) error { } func (c *BucketService) ListUserBuckets(req BucketListUserBucketsReq) (*BucketListUserBucketsResp, error) { - return JSONAPI[*BucketListUserBucketsResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketListUserBucketsResp{}) } diff --git a/sdks/storage/cdsapi/cache.go b/sdks/storage/cdsapi/cache.go index bb28e58..b71725d 100644 --- a/sdks/storage/cdsapi/cache.go +++ b/sdks/storage/cdsapi/cache.go @@ -26,5 +26,5 @@ func (r *CacheMovePackageResp) ParseResponse(resp *http.Response) error { } func (c *Client) CacheMovePackage(req CacheMovePackageReq) (*CacheMovePackageResp, error) { - return JSONAPI[*CacheMovePackageResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &CacheMovePackageResp{}) } diff --git a/sdks/storage/cdsapi/hub.go b/sdks/storage/cdsapi/hub.go index fb360a4..20a9d18 100644 --- a/sdks/storage/cdsapi/hub.go +++ b/sdks/storage/cdsapi/hub.go @@ -26,5 +26,5 @@ func (r *HubGetHubsResp) ParseResponse(resp *http.Response) error { } func (c *Client) HubGetHubs(req HubGetHubsReq) (*HubGetHubsResp, error) { - return JSONAPI[*HubGetHubsResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &HubGetHubsResp{}) } diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 9b4b8ed..aca0aa9 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -55,7 +55,7 @@ func (r *ObjectListByPathResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { - return JSONAPI[*ObjectListByPathResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectListByPathResp{}) } const ObjectListByIDsPath = "/object/listByIDs" @@ -78,7 +78,7 @@ func (r *ObjectListByIDsResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) { - return JSONAPI[*ObjectListByIDsResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectListByIDsResp{}) } const ObjectUploadPath = "/object/upload" @@ -108,6 +108,10 @@ type ObjectUploadResp struct { } func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { + type uploadInfo struct { + Info string `url:"info"` + } + url, err := url.JoinPath(c.cfg.URL, ObjectUploadPath) if err != nil { return nil, err @@ -119,7 +123,7 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { } resp, err := PostMultiPart(c.cfg, url, - map[string]string{"info": string(infoJSON)}, + uploadInfo{Info: string(infoJSON)}, iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { return &http2.IterMultiPartFile{ FieldName: "files", @@ -305,7 +309,7 @@ func (r *ObjectUpdateInfoResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) { - return JSONAPI[*ObjectUpdateInfoResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectUpdateInfoResp{}) } const ObjectUpdateInfoByPathPath = "/object/updateInfoByPath" @@ -328,7 +332,7 @@ func (r *ObjectUpdateInfoByPathResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) { - return JSONAPI[*ObjectUpdateInfoByPathResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectUpdateInfoByPathResp{}) } const ObjectMovePath = "/object/move" @@ -362,7 +366,7 @@ func (r *ObjectMoveResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) { - return JSONAPI[*ObjectMoveResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectMoveResp{}) } const ObjectDeletePath = "/object/delete" @@ -383,7 +387,7 @@ func (r *ObjectDeleteResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) Delete(req ObjectDelete) error { - return JSONAPINoData[*ObjectDeleteResp](c.cfg, http.DefaultClient, &req) + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const ObjectDeleteByPathPath = "/object/deleteByPath" @@ -405,7 +409,7 @@ func (r *ObjectDeleteByPathResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error { - return JSONAPINoData[*ObjectDeleteByPathResp](c.cfg, http.DefaultClient, &req) + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const ObjectClonePath = "/object/clone" @@ -434,7 +438,7 @@ func (r *ObjectCloneResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) { - return JSONAPI[*ObjectCloneResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectCloneResp{}) } const ObjectGetPackageObjectsPath = "/object/getPackageObjects" @@ -457,7 +461,7 @@ func (r *ObjectGetPackageObjectsResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) { - return JSONAPI[*ObjectGetPackageObjectsResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectGetPackageObjectsResp{}) } const ObjectNewMultipartUploadPath = "/v1/object/newMultipartUpload" @@ -481,7 +485,7 @@ func (r *ObjectNewMultipartUploadResp) ParseResponse(resp *http.Response) error } func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) { - return JSONAPI[*ObjectNewMultipartUploadResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectNewMultipartUploadResp{}) } const ObjectUploadPartPath = "/v1/object/uploadPart" @@ -560,5 +564,5 @@ func (r *ObjectCompleteMultipartUploadResp) ParseResponse(resp *http.Response) e } func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) { - return JSONAPI[*ObjectCompleteMultipartUploadResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectCompleteMultipartUploadResp{}) } diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index 621f64b..6c1d2c3 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -41,7 +41,7 @@ func (r *PackageGetResp) ParseResponse(resp *http.Response) error { } func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { - return JSONAPI[*PackageGetResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetResp{}) } const PackageGetByFullNamePath = "/package/getByFullName" @@ -65,7 +65,7 @@ func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error { } func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { - return JSONAPI[*PackageGetByFullNameResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetByFullNameResp{}) } const PackageCreatePath = "/package/create" @@ -89,7 +89,7 @@ func (r *PackageCreateResp) ParseResponse(resp *http.Response) error { } func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) { - return JSONAPI[*PackageCreateResp](s.cfg, http.DefaultClient, &req) + return JSONAPI(s.cfg, http.DefaultClient, &req, &PackageCreateResp{}) } const PackageCreateLoadPath = "/package/createLoad" @@ -164,7 +164,7 @@ func (r *PackageDeleteResp) ParseResponse(resp *http.Response) error { } func (c *PackageService) Delete(req PackageDelete) error { - return JSONAPINoData[*PackageDeleteResp](c.cfg, http.DefaultClient, &req) + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const PackageClonePath = "/package/clone" @@ -189,7 +189,7 @@ func (r *PackageCloneResp) ParseResponse(resp *http.Response) error { } func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) { - return JSONAPI[*PackageCloneResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageCloneResp{}) } const PackageListBucketPackagesPath = "/package/listBucketPackages" @@ -212,7 +212,7 @@ func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error } func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { - return JSONAPI[*PackageListBucketPackagesResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageListBucketPackagesResp{}) } const PackageGetCachedStoragesPath = "/package/getCachedStorages" @@ -235,5 +235,5 @@ func (r *PackageGetCachedStoragesResp) ParseResponse(resp *http.Response) error } func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { - return JSONAPI[*PackageGetCachedStoragesResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetCachedStoragesResp{}) } diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index b1986b8..22ad944 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -27,7 +27,7 @@ func (r *StorageLoadPackageResp) ParseResponse(resp *http.Response) error { } func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPackageResp, error) { - return JSONAPI[*StorageLoadPackageResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageLoadPackageResp{}) } const StorageCreatePackagePath = "/storage/createPackage" @@ -54,7 +54,7 @@ func (r *StorageCreatePackageResp) ParseResponse(resp *http.Response) error { } func (c *Client) StorageCreatePackage(req StorageCreatePackageReq) (*StorageCreatePackageResp, error) { - return JSONAPI[*StorageCreatePackageResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageCreatePackageResp{}) } const StorageGetPath = "/storage/get" @@ -77,5 +77,5 @@ func (r *StorageGetResp) ParseResponse(resp *http.Response) error { } func (c *Client) StorageGet(req StorageGet) (*StorageGetResp, error) { - return JSONAPI[*StorageGetResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageGetResp{}) } diff --git a/sdks/storage/cdsapi/user.go b/sdks/storage/cdsapi/user.go index 295e322..2230451 100644 --- a/sdks/storage/cdsapi/user.go +++ b/sdks/storage/cdsapi/user.go @@ -26,7 +26,7 @@ func (r *UserCreateResp) ParseResponse(resp *http.Response) error { } func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { - return JSONAPI[*UserCreateResp](c.cfg, http.DefaultClient, req) + return JSONAPI(c.cfg, http.DefaultClient, req, &UserCreateResp{}) } const UserDeletePath = "/v1/user/delete" @@ -46,5 +46,5 @@ func (r *UserDeleteResp) ParseResponse(resp *http.Response) error { } func (c *Client) UserDelete(req *UserDelete) error { - return JSONAPINoData[*UserDeleteResp](c.cfg, http.DefaultClient, req) + return JSONAPINoData(c.cfg, http.DefaultClient, req) } diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 4ba07df..988bb67 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -44,33 +44,32 @@ func ParseJSONResponse[TBody any](resp *http.Response) (TBody, error) { return ret, fmt.Errorf("unknow response content type: %s, status: %d, body(prefix): %s", contType, resp.StatusCode, strCont[:math2.Min(len(strCont), 200)]) } -func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) (Resp, error) { - var re Resp +func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req, resp Resp) (Resp, error) { param := req.MakeParam() httpReq, err := param.MakeRequest(cfg.URL) if err != nil { - return re, err + return resp, err } if cfg.AccessKey != "" && cfg.SecretKey != "" { err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) if err != nil { - return re, err + return resp, err } } - resp, err := cli.Do(httpReq) + httpResp, err := cli.Do(httpReq) if err != nil { - return re, err + return resp, err } - err = re.ParseResponse(resp) - return re, err + err = resp.ParseResponse(httpResp) + return resp, err } -func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) error { +func JSONAPINoData[Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) error { param := req.MakeParam() httpReq, err := param.MakeRequest(cfg.URL) From eacc4f6bd344c01199a70cb45c15e9bdf447ee42 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 14:35:02 +0800 Subject: [PATCH 072/155] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/sdks.go | 19 +++++----- sdks/storage/cdsapi/signer.go | 7 ++-- sdks/storage/cdsapi/storage_test.go | 56 +++++++++++++++++++++++++++++ sdks/storage/cdsapi/utils.go | 10 ++++-- 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/sdks/sdks.go b/sdks/sdks.go index 857c6b7..6f303c9 100644 --- a/sdks/sdks.go +++ b/sdks/sdks.go @@ -46,7 +46,10 @@ func (p *RequestParam) MakeRequest(baseURL string) (*http.Request, error) { } req.ContentLength = bodyLen req.URL.RawQuery = p.Query.Encode() - req.Header = p.Header + + if p.Header != nil { + req.Header = p.Header + } return req, nil } @@ -87,11 +90,11 @@ type StringBody struct { Value string } -func (s StringBody) Length() int64 { +func (s *StringBody) Length() int64 { return int64(len(s.Value)) } -func (s StringBody) IntoStream() io.ReadCloser { +func (s *StringBody) IntoStream() io.ReadCloser { return io.NopCloser(bytes.NewReader([]byte(s.Value))) } @@ -99,11 +102,11 @@ type BytesBody struct { Value []byte } -func (b BytesBody) Length() int64 { +func (b *BytesBody) Length() int64 { return int64(len(b.Value)) } -func (b BytesBody) IntoStream() io.ReadCloser { +func (b *BytesBody) IntoStream() io.ReadCloser { return io.NopCloser(bytes.NewReader(b.Value)) } @@ -112,11 +115,11 @@ type StreamBody struct { LengthHint int64 // 长度提示，如果长度未知，可以设置为-1 } -func (s StreamBody) Length() int64 { +func (s *StreamBody) Length() int64 { return s.LengthHint } -func (s StreamBody) IntoStream() io.ReadCloser { +func (s *StreamBody) IntoStream() io.ReadCloser { return s.Stream } @@ -134,7 +137,7 @@ func MakeJSONParam(method string, path string, body any) *RequestParam { return &RequestParam{ Method: method, Path: path, - Body: BytesBody{Value: data}, + Body: &BytesBody{Value: data}, } } diff --git a/sdks/storage/cdsapi/signer.go b/sdks/storage/cdsapi/signer.go index b4f8975..d0bea77 100644 --- a/sdks/storage/cdsapi/signer.go +++ b/sdks/storage/cdsapi/signer.go @@ -41,6 +41,9 @@ func Sign(req *http.Request, accessKey, secretKey string) error { hasher := sha256.New() hasher.Write(data) payloadHash = hex.EncodeToString(hasher.Sum(nil)) + } else { + hash := sha256.Sum256([]byte("")) + payloadHash = hex.EncodeToString(hash[:]) } signer := v4.NewSigner() @@ -52,9 +55,7 @@ func Sign(req *http.Request, accessKey, secretKey string) error { return nil } -// 对一个请求进行签名，并将签名信息添加到请求头中。 -// -// 不计算请求体的哈希，适合上传文件接口。 +// 对一个请求进行签名，不计算请求体的哈希，适合上传文件接口。 func SignWithoutBody(req *http.Request, accessKey, secretKey string) error { prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") cred, err := prod.Retrieve(context.TODO()) diff --git a/sdks/storage/cdsapi/storage_test.go b/sdks/storage/cdsapi/storage_test.go index edc1994..c61ad46 100644 --- a/sdks/storage/cdsapi/storage_test.go +++ b/sdks/storage/cdsapi/storage_test.go @@ -245,3 +245,59 @@ func Test_Cache(t *testing.T) { So(err, ShouldBeNil) }) } + +func Test_Sign(t *testing.T) { + Convey("签名接口", t, func() { + cli := NewClient(&Config{ + URL: "http://localhost:7890/v1", + AccessKey: "123456", + SecretKey: "123456", + }) + + fileData := make([]byte, 4096) + for i := 0; i < len(fileData); i++ { + fileData[i] = byte(i) + } + + pkgName := uuid.NewString() + createResp, err := cli.Package().Create(PackageCreate{ + UserID: 1, + BucketID: 1, + Name: pkgName, + }) + So(err, ShouldBeNil) + + _, err = cli.Object().Upload(ObjectUpload{ + ObjectUploadInfo: ObjectUploadInfo{ + UserID: 1, + PackageID: createResp.Package.PackageID, + }, + Files: iterator.Array( + &UploadingObject{ + Path: "abc/test", + File: io.NopCloser(bytes.NewBuffer(fileData)), + }, + &UploadingObject{ + Path: "test4", + File: io.NopCloser(bytes.NewBuffer(fileData)), + }, + ), + }) + So(err, ShouldBeNil) + + getResp, err := cli.Package().Get(PackageGetReq{ + UserID: 1, + PackageID: createResp.Package.PackageID, + }) + So(err, ShouldBeNil) + + So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID) + So(getResp.Package.Name, ShouldEqual, pkgName) + + err = cli.Package().Delete(PackageDelete{ + UserID: 1, + PackageID: createResp.Package.PackageID, + }) + So(err, ShouldBeNil) + }) +} diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 988bb67..811c225 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -54,7 +54,7 @@ func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http. } if cfg.AccessKey != "" && cfg.SecretKey != "" { - err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) + err = SignWithPayloadHash(httpReq, calcSha256(param.Body), cfg.AccessKey, cfg.SecretKey) if err != nil { return resp, err } @@ -78,7 +78,7 @@ func JSONAPINoData[Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) } if cfg.AccessKey != "" && cfg.SecretKey != "" { - err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) + err = SignWithPayloadHash(httpReq, calcSha256(param.Body), cfg.AccessKey, cfg.SecretKey) if err != nil { return err } @@ -103,8 +103,12 @@ func calcSha256(body sdks.RequestBody) string { hasher.Write(body.Value) return hex.EncodeToString(hasher.Sum(nil)) - default: + case *sdks.StreamBody: return "" + + default: + hash := sha256.Sum256([]byte("")) + return hex.EncodeToString(hash[:]) } } From 1589ed240695c5dcc4d56f7f470a0f906e6490bb Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 16:19:02 +0800 Subject: [PATCH 073/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=80=9A=E8=BF=87ID?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E6=96=87=E4=BB=B6=E7=9A=84=E9=A2=84=E7=AD=BE?= =?UTF-8?q?=E5=90=8D=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/presigned.go | 15 ++++++++++- sdks/storage/cdsapi/presigned_test.go | 38 ++++++++++++++++++++++----- sdks/storage/cdsapi/signer.go | 2 +- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go index 5e755fb..b357e71 100644 --- a/sdks/storage/cdsapi/presigned.go +++ b/sdks/storage/cdsapi/presigned.go @@ -33,10 +33,23 @@ type PresignedObjectDownloadByPath struct { Length *int64 `form:"length" url:"length,omitempty"` } -func (c *PresignedService) ObjectDownload(req PresignedObjectDownloadByPath, expireIn int) (string, error) { +func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) { return c.presign(req, PresignedObjectDownloadByPathPath, http.MethodGet, expireIn) } +const PresignedObjectDownloadPath = "/v1/presigned/object/download" + +type PresignedObjectDownload struct { + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `form:"objectID" url:"objectID" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectDownloadPath, http.MethodGet, expireIn) +} + const PresignedObjectUploadPath = "/v1/presigned/object/upload" type PresignedObjectUpload struct { diff --git a/sdks/storage/cdsapi/presigned_test.go b/sdks/storage/cdsapi/presigned_test.go index 37c5b87..f4071c9 100644 --- a/sdks/storage/cdsapi/presigned_test.go +++ b/sdks/storage/cdsapi/presigned_test.go @@ -16,7 +16,7 @@ func Test_Presigned(t *testing.T) { Convey("下载文件", t, func() { pre := cli.Presigned() - url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ + url, err := pre.ObjectDownloadByPath(PresignedObjectDownloadByPath{ UserID: 1, PackageID: 3, Path: "example.java", @@ -39,14 +39,16 @@ func Test_Presigned(t *testing.T) { }) } -func Test_PresignedObjectDownload(t *testing.T) { +func Test_PresignedObjectDownloadByPath(t *testing.T) { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", }) Convey("下载文件", t, func() { pre := cli.Presigned() - url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ + url, err := pre.ObjectDownloadByPath(PresignedObjectDownloadByPath{ UserID: 1, PackageID: 3, Path: "example.java", @@ -58,6 +60,26 @@ func Test_PresignedObjectDownload(t *testing.T) { }) } +func Test_PresignedObjectDownload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectDownload(PresignedObjectDownload{ + UserID: 1, + ObjectID: 1039, + // Offset: 1, + // Length: types.Ref(int64(100)), + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + func Test_PresignedObjectUpload(t *testing.T) { cli := NewClient(&Config{ URL: "http://localhost:7890", @@ -94,7 +116,9 @@ func Test_PresignedNewMultipartUpload(t *testing.T) { func Test_PresignedObjectUploadPart(t *testing.T) { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", }) Convey("上传分片", t, func() { @@ -111,7 +135,9 @@ func Test_PresignedObjectUploadPart(t *testing.T) { func Test_PresignedCompleteMultipartUpload(t *testing.T) { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", }) Convey("合并分片", t, func() { diff --git a/sdks/storage/cdsapi/signer.go b/sdks/storage/cdsapi/signer.go index d0bea77..67e39e5 100644 --- a/sdks/storage/cdsapi/signer.go +++ b/sdks/storage/cdsapi/signer.go @@ -95,7 +95,7 @@ func SignWithPayloadHash(req *http.Request, payloadHash string, accessKey, secre // // expiration为签名过期时间，单位为秒。 // -// 签名时不会包含请求体的哈希值。 +// 签名时不会包含请求体的哈希值。注：不要设置任何额外的Header（除了自动添加的Host），以免签名校验不通过 func Presign(req *http.Request, accessKey, secretKey string, expiration int) (string, error) { query := req.URL.Query() query.Add("X-Expires", fmt.Sprintf("%v", expiration)) From fe198e76bfb40bdbff88e7bf1f38239fa5322161 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 11 Mar 2025 16:27:40 +0800 Subject: [PATCH 074/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 8 ++++---- sdks/pcmscheduler/models.go | 15 ++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index e4d7cde..be1e3ae 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -58,10 +58,10 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) } type CreateJobReq struct { - Name string `json:"name"` - Description string `json:"description"` - JobResources schsdk.JobResources `json:"jobResources"` - DataDistribute DataDistribute `json:"dataDistributes"` + Name string `json:"name"` + Description string `json:"description"` + JobResources schsdk.JobResources `json:"jobResources"` + DataDistributes DataDistribute `json:"dataDistributes"` } type CommonJsonData struct { diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index daca965..a993342 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -385,13 +385,14 @@ func (d *DataBindingBase) Noop() {} type DatasetBinding struct { serder.Metadata `union:"dataset"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - PackageID cdssdk.PackageID `json:"packageID"` - RepositoryName string `json:"repositoryName"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints float64 `json:"points"` } type ModelBinding struct { From fdace7b42b8c9866ad1966a7c2f6dc441cc96e34 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 13 Mar 2025 09:17:24 +0800 Subject: [PATCH 075/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index a993342..1b3eaff 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -420,6 +420,7 @@ type CodeBinding struct { ImageID schsdk.ImageID `json:"imageID"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` // 当集群为openi的时候，需要传入分支 Branch string `json:"branch"` } From a936cffa134eece2e32229f4ff2e39f592777d1b Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 13 Mar 2025 16:59:03 +0800 Subject: [PATCH 076/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=9B=E5=BB=BApac?= =?UTF-8?q?kage=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 6ba148c..7393c28 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -126,8 +126,9 @@ var JobResourceTypeUnion = types.NewTypeUnion[JobResource]( (*NPU)(nil), (*MLU)(nil), (*DCU)(nil), - (*Memory)(nil), + (*MEMORY)(nil), (*PRICE)(nil), + (*STORAGE)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type") @@ -144,6 +145,14 @@ type CPU struct { Number int64 `json:"number"` } +type STORAGE struct { + serder.Metadata `union:"STORAGE"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + type GPU struct { serder.Metadata `union:"GPU"` JobResourceBase @@ -160,8 +169,8 @@ type NPU struct { Number int64 `json:"number"` } -type Memory struct { - serder.Metadata `union:"Memory"` +type MEMORY struct { + serder.Metadata `union:"MEMORY"` JobResourceBase Type string `json:"type"` Name string `json:"name"` From 4374600529fdec8617b4ef92fc065ed47ee0f1bd Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 14:23:47 +0800 Subject: [PATCH 077/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B6=85=E7=AE=97?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/client.go | 63 +++++ sdks/hpc/config.go | 5 + sdks/hpc/job.go | 71 ++++++ sdks/hpc/models.go | 525 +++++++++++++++++++++++++++++++++++++++ sdks/scheduler/models.go | 21 ++ 5 files changed, 685 insertions(+) create mode 100644 sdks/hpc/client.go create mode 100644 sdks/hpc/config.go create mode 100644 sdks/hpc/job.go create mode 100644 sdks/hpc/models.go diff --git a/sdks/hpc/client.go b/sdks/hpc/client.go new file mode 100644 index 0000000..07b4393 --- /dev/null +++ b/sdks/hpc/client.go @@ -0,0 +1,63 @@ +package hpc + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type respons2[T any] struct { + Code int `json:"code"` + Message string `json:"msg"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/hpc/config.go b/sdks/hpc/config.go new file mode 100644 index 0000000..d168c41 --- /dev/null +++ b/sdks/hpc/config.go @@ -0,0 +1,5 @@ +package hpc + +type Config struct { + URL string `json:"url"` +} diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go new file mode 100644 index 0000000..ddc5205 --- /dev/null +++ b/sdks/hpc/job.go @@ -0,0 +1,71 @@ +package hpc + +import ( + "fmt" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" +) + +type CreateJobReq struct { + Name string `json:"name"` + ClusterID schsdk.ClusterID `json:"clusterId"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + Parameters HPCParameter `json:"parameters"` +} + +type HPCParameter struct { + JobName string `json:"jobName"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + InputFile string `json:"inputFile"` +} + +type CreateJobResp struct { + Backend string `json:"backend"` + JobInfo HPCJobInfo `json:"jobInfo"` +} + +type HPCJobInfo struct { + JobDir string `json:"jobDir"` + JobID string `json:"jobId"` +} + +func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/hpc/commitHpcTask") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, fmt.Errorf("error: %s", codeResp.Message) + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) + +} diff --git a/sdks/hpc/models.go b/sdks/hpc/models.go new file mode 100644 index 0000000..6511bb9 --- /dev/null +++ b/sdks/hpc/models.go @@ -0,0 +1,525 @@ +package hpc + +import ( + "gitlink.org.cn/cloudream/common/pkgs/types" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/serder" + "time" +) + +type ResourceType string + +const ( + ResourceTypeCPU ResourceType = "CPU" + ResourceTypeNPU ResourceType = "NPU" + ResourceTypeGPU ResourceType = "GPU" + ResourceTypeMLU ResourceType = "MLU" + ResourceTypeStorage ResourceType = "STORAGE" + ResourceTypeMemory ResourceType = "MEMORY" + + Split = "/" + + CODE = "code" + DATASET = "dataset" + IMAGE = "image" + MODEL = "model" + RESULT = "result" + + OrderByName = "name" + OrderBySize = "size" + OrderByTime = "time" + + StorageTypeURL = "url" + StorageTypeJCS = "jcs" + + RejectedStatus = "rejected" + PendingStatus = "pending" + ApprovedStatus = "approved" + RevokedStatus = "revoked" + CancelStatus = "cancel" + ExpiredStatus = "expired" + + ApplyAccess = "apply" + PrivateAccess = "private" + PublicAccess = "public" + + PreferencePriority = "preference" + SpecifyClusterPriority = "specify" + + FailedStatus = "failed" + SuccessStatus = "success" + + Query = "query" + Delete = "delete" + + ChildrenType = "children" + ParentType = "parent" + + PlatformSugon = "sugon" + PlatformOpenI = "OpenI" + PlatformModelArts = "ModelArts" + + URL = "url" + ID = "id" + + Startup = "startup" +) + +type TaskID int64 +type DataID int64 + +type ClusterDetail struct { + // 集群ID + ClusterId schsdk.ClusterID `json:"clusterID"` + // 集群功能类型：云算，智算，超算 + ClusterType string `json:"clusterType"` + // 集群地区：华东地区、华南地区、华北地区、华中地区、西南地区、西北地区、东北地区 + Region string `json:"region"` + // 资源类型 + Resources2 []ResourceData `json:"resources1,omitempty"` + //Resources2 []ResourceData `json:"resources"` + Resources []ClusterResource `json:"resources"` +} + +type ClusterResource struct { + Resource TmpResourceData `json:"resource"` + BaseResources []TmpResourceData `json:"baseResources"` +} + +type TmpResourceData struct { + Type ResourceType `json:"type"` + Name string `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceData interface { + Noop() +} + +var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( + (*CPUResourceData)(nil), + (*NPUResourceData)(nil), + (*GPUResourceData)(nil), + (*MLUResourceData)(nil), + (*DCUResourceData)(nil), + (*GCUResourceData)(nil), + (*GPGPUResourceData)(nil), + (*StorageResourceData)(nil), + (*MemoryResourceData)(nil), + (*BalanceResourceData)(nil), + (*RateResourceData)(nil), +) +var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "type") + +type ResourceDataBase struct{} + +func (d *ResourceDataBase) Noop() {} + +type UnitValue[T any] struct { + Unit string `json:"unit"` + Value T `json:"value"` +} + +type CPUResourceData struct { + serder.Metadata `union:"CPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type NPUResourceData struct { + serder.Metadata `union:"NPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPUResourceData struct { + serder.Metadata `union:"GPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type MLUResourceData struct { + serder.Metadata `union:"MLU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type DCUResourceData struct { + serder.Metadata `union:"DCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GCUResourceData struct { + serder.Metadata `union:"GCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPGPUResourceData struct { + serder.Metadata `union:"ILUVATAR-GPGPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type StorageResourceData struct { + serder.Metadata `union:"STORAGE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type MemoryResourceData struct { + serder.Metadata `union:"MEMORY"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type BalanceResourceData struct { + serder.Metadata `union:"BALANCE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type RateResourceData struct { + serder.Metadata `union:"RATE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceRange struct { + UserID cdssdk.UserID `json:"userID"` + Type ResourceType `json:"type"` + GPU Range `json:"gpu"` + GPUNumber int `json:"gpuNumber"` + CPU Range `json:"cpu"` + Memory Range `json:"memory"` + Storage Range `json:"storage"` +} + +type Range struct { + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +type ResourcePriority interface { + Noop() +} + +type ResourcePriorityBase struct { +} + +var ResourcePriorityTypeUnion = types.NewTypeUnion[ResourcePriority]( + (*RegionPriority)(nil), + (*ChipPriority)(nil), + (*BiasPriority)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&ResourcePriorityTypeUnion, "type") + +func (d *ResourcePriorityBase) Noop() {} + +type RegionPriority struct { + serder.Metadata `union:"region"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type ChipPriority struct { + serder.Metadata `union:"chip"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type BiasPriority struct { + serder.Metadata `union:"bias"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type TaskMessage struct { + Status string `json:"status"` + Message string `json:"message"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` +} + +type UploadParams struct { + DataType string `json:"dataType"` + UploadInfo UploadInfo `json:"uploadInfo"` +} + +type UploadInfo interface { + Noop() +} + +var UploadInfoTypeUnion = types.NewTypeUnion[UploadInfo]( + (*LocalUploadInfo)(nil), + (*RemoteUploadInfo)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type") + +type LocalUploadInfo struct { + serder.Metadata `union:"local"` + UploadInfoBase + Type string `json:"type"` + LocalPath string `json:"localPath"` + ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` +} + +type RemoteUploadInfo struct { + serder.Metadata `union:"url"` + UploadInfoBase + Type string `json:"type"` + Url string `json:"url"` + Branch string `json:"branch"` + DataName string `json:"dataName"` + Cluster schsdk.ClusterID `json:"clusterID"` +} + +type UploadInfoBase struct{} + +func (d *UploadInfoBase) Noop() {} + +type UploadPriority interface { + Noop() +} + +var UploadPriorityTypeUnion = types.NewTypeUnion[UploadPriority]( + (*Preferences)(nil), + (*SpecifyCluster)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadPriorityTypeUnion, "type") + +type Preferences struct { + serder.Metadata `union:"preference"` + UploadPriorityBase + Type string `json:"type"` + ResourcePriorities []ResourcePriority `json:"priorities"` +} + +type SpecifyCluster struct { + serder.Metadata `union:"specify"` + UploadPriorityBase + Type string `json:"type"` + Clusters []schsdk.ClusterID `json:"clusters"` +} + +type UploadPriorityBase struct{} + +func (d *UploadPriorityBase) Noop() {} + +type QueryData struct { + DataType string `json:"dataType" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Path string `json:"path"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + CurrentPage int `json:"currentPage" binding:"required"` + PageSize int `json:"pageSize" binding:"required"` + OrderBy string `json:"orderBy" binding:"required"` +} + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*DatasetBinding)(nil), + (*ModelBinding)(nil), + (*CodeBinding)(nil), + (*ImageBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct{} + +func (d *DataBindingBase) Noop() {} + +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints float64 `json:"points"` +} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` +} + +type CodeBinding struct { + serder.Metadata `union:"code"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Description string `json:"description"` + ImageID schsdk.ImageID `json:"imageID"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` + // 当集群为openi的时候，需要传入分支 + Branch string `json:"branch"` +} + +//type ImageBinding struct { +// serder.Metadata `union:"image"` +// DataBindingBase +// Type string `json:"type"` +// Name string `json:"name"` +// ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` +// Description string `json:"description"` +// Architecture string `json:"architecture"` +// ResourceType string `json:"resourceType"` +// Tags []string `json:"tags"` +// PackageID cdssdk.PackageID `json:"packageID"` +//} + +type ImageBinding struct { + serder.Metadata `union:"image"` + DataBindingBase + Type string `json:"type"` + ID int64 `json:"id"` + Name string `json:"name"` + IDType string `json:"idType"` + ImageID string `json:"imageID"` + ClusterID schsdk.ClusterID `json:"clusterID"` +} + +type Image struct { + ImageID schsdk.ImageID `json:"imageID" gorm:"column:ImageID;primaryKey"` + Name string `json:"name" gorm:"column:Name"` + CreateTime time.Time `json:"createTime" gorm:"column:CreateTime"` + ClusterImage []ClusterImage `gorm:"foreignKey:image_id;references:ImageID" json:"clusterImages"` +} + +type ClusterImage struct { + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + OriginImageType string `gorm:"column:origin_image_type" json:"originImageType"` + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + OriginImageName string `gorm:"column:origin_image_name" json:"originImageName"` + ClusterImageCard []ClusterImageCard `gorm:"foreignKey:origin_image_id;references:origin_image_id" json:"cards"` +} + +func (ClusterImage) TableName() string { + return "clusterImage" +} + +type ClusterImageCard struct { + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + Card string `gorm:"column:card" json:"card"` +} + +func (ClusterImageCard) TableName() string { + return "clusterImageCard" +} + +type QueryBindingFilters struct { + Status string `json:"status"` + Name string `json:"name"` +} + +type QueryBindingDataParam interface { + Noop() +} + +var QueryBindingDataParamTypeUnion = types.NewTypeUnion[QueryBindingDataParam]( + (*PrivateLevel)(nil), + (*ApplyLevel)(nil), + (*PublicLevel)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&QueryBindingDataParamTypeUnion, "type") + +type QueryBindingDataParamBase struct{} + +func (d *QueryBindingDataParamBase) Noop() {} + +type PrivateLevel struct { + serder.Metadata `union:"private"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + BindingID int64 `json:"bindingID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type ApplyLevel struct { + serder.Metadata `union:"apply"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type PublicLevel struct { + serder.Metadata `union:"public"` + QueryBindingDataParamBase + UserID cdssdk.UserID `json:"userID" binding:"required"` + Type string `json:"type" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 7393c28..45026f2 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -60,6 +60,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*FinetuningJobInfo)(nil), (*DataPreprocessJobInfo)(nil), (*PCMJobInfo)(nil), + (*HPCJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -92,6 +93,26 @@ type PCMJobInfo struct { JobResources JobResources `json:"jobResources"` } +type HPCJobInfo struct { + serder.Metadata `union:"HPC"` + JobInfoBase + Type string `json:"type"` + Name string `json:"name"` + ClusterID ClusterID `json:"clusterID"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + Parameters HPCParameter `json:"parameters"` +} + +type HPCParameter struct { + JobName string `json:"jobName"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + InputFile string `json:"inputFile"` +} + type JobResources struct { //任务分配策略：负载均衡、积分优先、随机分配等，dataLocality, leastLoadFirst ScheduleStrategy string `json:"scheduleStrategy"` From a460daa8614e99f9102c40f810257d4b502bd17a Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 15:30:07 +0800 Subject: [PATCH 078/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E5=AD=90=E7=AE=97=E6=B3=95=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 0637b8c..a20cf30 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -166,6 +166,7 @@ type PackageCloneParam struct { Description string `json:"description"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` ImageID schsdk.ImageID `json:"imageID"` } From 48681fa14707fde4818e780ce876ebd5ec584e09 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 14 Mar 2025 16:59:12 +0800 Subject: [PATCH 079/155] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BB=8EStorage?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0Package=E7=9A=84=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index 22ad944..d275432 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -46,7 +46,7 @@ func (r *StorageCreatePackageReq) MakeParam() *sdks.RequestParam { } type StorageCreatePackageResp struct { - PackageID cdssdk.PackageID `json:"packageID"` + Package cdssdk.Package `json:"package"` } func (r *StorageCreatePackageResp) ParseResponse(resp *http.Response) error { From e4a9171da2241341899ea6df0c46defe2198b826 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 17:47:11 +0800 Subject: [PATCH 080/155] =?UTF-8?q?=E8=B6=85=E7=AE=97=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 3 ++- sdks/scheduler/models.go | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index ddc5205..363be1a 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -15,7 +15,8 @@ type CreateJobReq struct { Backend string `json:"backend"` App string `json:"app"` OperateType string `json:"operateType"` - Parameters HPCParameter `json:"parameters"` + //Parameters HPCParameter `json:"parameters"` + Parameters map[string]string `json:"parameters"` } type HPCParameter struct { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 45026f2..9051489 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -106,11 +106,17 @@ type HPCJobInfo struct { } type HPCParameter struct { - JobName string `json:"jobName"` - Partition string `json:"partition"` - Ntasks string `json:"ntasks"` - Nodes string `json:"nodes"` - InputFile string `json:"inputFile"` + JobName string `json:"jobName"` + JobDir string `json:"jobDir"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` +} + +type HPCBindingFile struct { + ParamName string `json:"paramName"` + BindingID int64 `json:"bindingID"` } type JobResources struct { From 1da627d0baf98717ef71c6066a4a5c5d9941a43f Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 18:47:48 +0800 Subject: [PATCH 081/155] fix --- sdks/storage/cdsapi/object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index aca0aa9..49c0b3a 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -66,7 +66,7 @@ type ObjectListByIDs struct { } func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { - return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) + return sdks.MakeQueryParam(http.MethodPost, ObjectListByIDsPath, r) } type ObjectListByIDsResp struct { From f5cad317d16abeb085dafaceb3db4a5fb93b376a Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 20:01:28 +0800 Subject: [PATCH 082/155] fix --- sdks/storage/cdsapi/object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 49c0b3a..aca0aa9 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -66,7 +66,7 @@ type ObjectListByIDs struct { } func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { - return sdks.MakeQueryParam(http.MethodPost, ObjectListByIDsPath, r) + return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) } type ObjectListByIDsResp struct { From 058904bff5240be8b0345b2a4ccde0c400ef0845 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 20:51:02 +0800 Subject: [PATCH 083/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=B6=85=E7=AE=97?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 1 + sdks/scheduler/models.go | 1 + 2 files changed, 2 insertions(+) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 363be1a..f860bc1 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -24,6 +24,7 @@ type HPCParameter struct { Partition string `json:"partition"` Ntasks string `json:"ntasks"` Nodes string `json:"nodes"` + BamFile string `json:"bamFile"` InputFile string `json:"inputFile"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9051489..4890e53 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -111,6 +111,7 @@ type HPCParameter struct { Partition string `json:"partition"` Ntasks string `json:"ntasks"` Nodes string `json:"nodes"` + BamFile string `json:"bamFile"` HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` } From 733345a5f9857da27239b9044056267ad2becbf1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 18 Mar 2025 11:23:34 +0800 Subject: [PATCH 084/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=95=B0=E9=87=8F=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 4890e53..ae87c13 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -65,13 +65,18 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` + LocalJobID string `json:"localJobID"` + TargetLocalJobID string `json:"targetLocalJobID"` } func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } +func (i *JobInfoBase) GetTargetLocalJobID() string { + return i.TargetLocalJobID +} + type NormalJobInfo struct { serder.Metadata `union:"Normal"` JobInfoBase From ce4cb96abe16f9a899c1a9657b13e8e4859d286c Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 18 Mar 2025 17:03:37 +0800 Subject: [PATCH 085/155] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 5 +++-- sdks/scheduler/models.go | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 9439b9c..bf5cd7b 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -49,8 +49,9 @@ const ( PreferencePriority = "preference" SpecifyClusterPriority = "specify" - FailedStatus = "failed" - SuccessStatus = "success" + FailedStatus = "failed" + SuccessStatus = "success" + UploadingStatus = "uploading" Query = "query" Delete = "delete" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index ae87c13..9b94602 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -49,6 +49,7 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string + GetTargetLocalJobID() []string } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -65,16 +66,16 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` - TargetLocalJobID string `json:"targetLocalJobID"` + LocalJobID string `json:"localJobID"` + TargetLocalJobIDs []string `json:"targetLocalJobIDs"` } func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } -func (i *JobInfoBase) GetTargetLocalJobID() string { - return i.TargetLocalJobID +func (i *JobInfoBase) GetTargetLocalJobID() []string { + return i.TargetLocalJobIDs } type NormalJobInfo struct { From f5a4b7176dd5d475532c42e9c7bbadc6d5dae7d3 Mon Sep 17 00:00:00 2001 From: qiwang <1364512070@qq.com> Date: Thu, 20 Mar 2025 16:48:51 +0800 Subject: [PATCH 086/155] feat:query taskResult --- sdks/pcmscheduler/models.go | 93 +++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index af991f5..bea1599 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -227,6 +227,99 @@ type ResourceRange struct { Ids []string `json:"ids"` } +type PCMJobData struct { + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + Param string `json:"param"` +} + +type JobInfo struct { + Name string `json:"name"` + Description string `json:"description,optional"` + JobResources JobResources `json:"jobResources"` + DataDistributeData DataDistributeData `json:"dataDistributes"` + ResultFile ResultFile `json:"resultFile"` +} + +type ResultFile struct { + Path string `json:"path"` + UpdateTime string `json:"updateTime"` + Size int `json:"size"` +} + +type DataDistributeData struct { + Dataset []DataItem `json:"dataset"` + Code []DataItem `json:"code"` + Image []DataItem `json:"image"` + Model []DataItem `json:"model"` +} + +// DataItem 定义 dataDistributes 中的数据项结构体 +type DataItem struct { + DataName string `json:"dataName"` + PackageID int `json:"packageID"` + PackageName string `json:"packageName"` + ResourceFile string `json:"resourceFile"` + Clusters []DataDetail `json:"clusters"` +} + +type JobResources struct { + ScheduleStrategy string `json:"scheduleStrategy"` + Clusters []*JobClusterInfo `json:"clusters"` +} + +type JobClusterInfo struct { + ClusterID string `json:"clusterID"` + Resources []map[string]interface{} `json:"resources,optional"` + Runtime JobRuntimeInfo `json:"runtime,optional"` +} + +type JobRuntimeInfo struct { + Command string `json:"command,optional"` + Envs map[string]interface{} `json:"envs,optional"` + Params map[string]interface{} `json:"params,optional"` +} + +// Engine 定义 engine 结构体 +type Engine struct { + EngineName string `json:"EngineName"` + EngineVersion string `json:"EngineVersion"` + ImageUrl string `json:"ImageUrl"` + InstallSysPackages bool `json:"InstallSysPackages"` +} + +// bindingCluster 定义根结构体 +type BindingParam struct { + Name string `json:"name"` + Description string `json:"description"` + Engine Engine `json:"engine"` + CodeDir string `json:"codeDir"` + BootFile string `json:"bootFile"` + Branch string `json:"branch"` + Platform string `json:"platform"` + UserID int `json:"userID"` + PackageId int `json:"packageId"` + BucketID int `json:"bucketID"` +} + +// bindingCluster 定义根结构体 +type BindingCluster struct { + BindingId int `json:"binding_id"` + ClusterId string `json:"cluster_id"` + Status string `json:"status"` + Param string `json:"param"` +} + +/*type TaskResult struct { + Name string `json:"name"` + Description string `json:"description"` + DataDistribute DataDistribute `json:"dataDistributes"` +} + +type ClusterResources struct { + ClusterId string `json:"clusterId"` +}*/ + type Range struct { Min float64 `json:"min"` Max float64 `json:"max"` From a7c1a55bf026d1e7f4395f6a86b1772679f9e5b1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 21 Mar 2025 11:14:11 +0800 Subject: [PATCH 087/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E5=92=8C=E5=9B=9E=E6=BA=90=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/job.go | 7 +++++ sdks/pcmscheduler/models.go | 3 ++ sdks/scheduler/models.go | 55 +++++++++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/job.go b/sdks/pcmscheduler/job.go index 0470ec8..fd1997b 100644 --- a/sdks/pcmscheduler/job.go +++ b/sdks/pcmscheduler/job.go @@ -8,9 +8,16 @@ import ( type PCMJob struct { ID string `gorm:"column:id" json:"ID"` + JobType string `gorm:"column:job_type" json:"jobType"` UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"` LocalJobID string `gorm:"column:local_job_id" json:"localJobID"` Param string `gorm:"column:param" json:"param"` CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` } + +type PCMJobDataReturn struct { + JobID string `gorm:"column:job_id" json:"jobID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` +} diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index bf5cd7b..2eb6f3f 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -63,6 +63,9 @@ const ( PlatformOpenI = "OpenI" PlatformModelArts = "ModelArts" + PlatformPCM = "PCM" + PlatformHPC = "HPC" + URL = "url" ID = "id" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9b94602..3faf9e2 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -8,6 +8,7 @@ import ( const ( JobTypeNormal = "Normal" + JobTypePCM = "PCM" JobTypeResource = "Resource" JobTypeInstance = "Instance" JobTypeFinetuning = "Finetuning" @@ -49,7 +50,8 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string - GetTargetLocalJobID() []string + GetTargetLocalJobIDs() []string + RemoveTargetLocalJobID(targetID string) } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -62,6 +64,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*DataPreprocessJobInfo)(nil), (*PCMJobInfo)(nil), (*HPCJobInfo)(nil), + (*BindingJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -74,10 +77,23 @@ func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } -func (i *JobInfoBase) GetTargetLocalJobID() []string { +func (i *JobInfoBase) GetTargetLocalJobIDs() []string { return i.TargetLocalJobIDs } +func (i *JobInfoBase) GetTargetLocalJobIDs2() []string { + return i.TargetLocalJobIDs +} + +func (i *JobInfoBase) RemoveTargetLocalJobID(targetID string) { + // 从i.TargetLocalJobIDs中删除id + for j, id := range i.TargetLocalJobIDs { + if targetID == id { + i.TargetLocalJobIDs = append(i.TargetLocalJobIDs[:j], i.TargetLocalJobIDs[j+1:]...) + } + } +} + type NormalJobInfo struct { serder.Metadata `union:"Normal"` JobInfoBase @@ -99,6 +115,41 @@ type PCMJobInfo struct { JobResources JobResources `json:"jobResources"` } +type BindingJobInfo struct { + serder.Metadata `union:"Binding"` + JobInfoBase + Type string `json:"type"` + Info DataBinding `json:"info"` +} + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*ModelBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct{} + +func (d *DataBindingBase) Noop() {} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + //Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + //PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` +} + type HPCJobInfo struct { serder.Metadata `union:"HPC"` JobInfoBase From 9682b7422b9962d0e31bebdabc366d35fe789dbb Mon Sep 17 00:00:00 2001 From: tzwang Date: Fri, 21 Mar 2025 17:23:04 +0800 Subject: [PATCH 088/155] added pcm inference job creation --- sdks/pcmscheduler/jobset.go | 44 ++++++++++++++++++++++++++++++++++--- sdks/pcmscheduler/models.go | 5 +++-- sdks/scheduler/models.go | 11 ++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index be1e3ae..0d9ea55 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -57,6 +57,10 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) return nil, fmt.Errorf("unknow response content type: %s", contType) } +type CreateInferenceJobResp struct { + TaskId string `json:"taskId"` +} + type CreateJobReq struct { Name string `json:"name"` Description string `json:"description"` @@ -121,8 +125,42 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } +func (c *Client) CreateInferenceJob(req CreateJobReq, token string) (*CreateInferenceJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateInferenceJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, fmt.Errorf("error: %s", codeResp.Message) + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) + +} + func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/createTask") + targetUrl, err := url.JoinPath(c.baseURL, "schedule/createTask") if err != nil { return nil, err } @@ -181,7 +219,7 @@ type DataScheduleResults struct { } func (c *Client) RunJob(req RunJobReq, token string) error { - targetUrl, err := url.JoinPath(c.baseURL, "runTask") + targetUrl, err := url.JoinPath(c.baseURL, "schedule/runTask") if err != nil { return err } @@ -220,7 +258,7 @@ type CancelJobReq struct { } func (c *Client) CancelJob(req CancelJobReq) error { - targetUrl, err := url.JoinPath(c.baseURL, "/queryResources") + targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") if err != nil { return err } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 2eb6f3f..d8642cf 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -63,8 +63,9 @@ const ( PlatformOpenI = "OpenI" PlatformModelArts = "ModelArts" - PlatformPCM = "PCM" - PlatformHPC = "HPC" + PlatformPCM = "PCM" + PlatformPCMInference = "PCM_Inference" + PlatformHPC = "HPC" URL = "url" ID = "id" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 3faf9e2..e676645 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -65,6 +65,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*PCMJobInfo)(nil), (*HPCJobInfo)(nil), (*BindingJobInfo)(nil), + (*PCMInferenceJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -105,6 +106,16 @@ type NormalJobInfo struct { ModelJobInfo ModelJobInfo `json:"modelJobInfo"` } +type PCMInferenceJobInfo struct { + serder.Metadata `union:"PCM_Inference"` + JobInfoBase + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` +} + type PCMJobInfo struct { serder.Metadata `union:"PCM"` JobInfoBase From f9b6e073764c21ad644ae3881665bc0b4626b5b1 Mon Sep 17 00:00:00 2001 From: qiwang <1364512070@qq.com> Date: Fri, 21 Mar 2025 17:35:05 +0800 Subject: [PATCH 089/155] feat:query taskResult --- go.mod | 2 +- sdks/pcmscheduler/models.go | 29 ++++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index fc87255..33c40ed 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/credentials v1.17.62 + github.com/google/go-querystring v1.1.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.15 @@ -33,7 +34,6 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-querystring v1.1.0 github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index bea1599..b898051 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -227,6 +227,12 @@ type ResourceRange struct { Ids []string `json:"ids"` } +type QueryTaskResultReq struct { + TaskID string `json:"taskID" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` +} + type PCMJobData struct { ID string `json:"id"` CreatedAt time.Time `json:"created_at"` @@ -242,9 +248,26 @@ type JobInfo struct { } type ResultFile struct { - Path string `json:"path"` - UpdateTime string `json:"updateTime"` - Size int `json:"size"` + CommonPrefixes []interface{} `json:"commonPrefixes"` + Objects []Object `json:"objects"` +} + +// Object 表示对象结构体 +type Object struct { + ObjectID int `json:"objectID"` + PackageID int `json:"packageID"` + Path string `json:"path"` + Size string `json:"size"` + FileHash string `json:"fileHash"` + Redundancy Redundancy `json:"redundancy"` + CreateTime time.Time `json:"createTime"` + UpdateTime time.Time `json:"updateTime"` +} + +// Redundancy 表示冗余信息结构体 +type Redundancy struct { + Type string `json:"type"` + RepCount int `json:"repCount"` } type DataDistributeData struct { From 54238a337c651177092bb133635fffe2f7689691 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 24 Mar 2025 14:52:52 +0800 Subject: [PATCH 090/155] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E8=B5=84=E6=BA=90URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 0d9ea55..e6b8b41 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -20,7 +20,7 @@ type GetClusterInfoReq struct { //} func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/queryResources") + targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") if err != nil { return nil, err } From 75f4426fb2ed24fd3b5ac540ee2b8c8085ff533f Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 26 Mar 2025 16:30:29 +0800 Subject: [PATCH 091/155] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E6=94=AF=E6=8C=81=E5=88=86=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index aca0aa9..718c3ba 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -33,12 +33,13 @@ func (c *Client) Object() *ObjectService { const ObjectListPathByPath = "/object/listByPath" type ObjectListByPath struct { - UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` - PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID"` - Path string `form:"path" url:"path"` // 允许为空字符串 - IsPrefix bool `form:"isPrefix" url:"isPrefix"` - NoRecursive bool `form:"noRecursive" url:"noRecursive"` // 仅当isPrefix为true时有效，表示仅查询直接属于Prefix下的对象，对于更深的对象，返回它们的公共前缀 - + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID" json:"userID"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID" json:"packageID"` + Path string `form:"path" url:"path" json:"path"` // 允许为空字符串 + IsPrefix bool `form:"isPrefix" url:"isPrefix" json:"isPrefix"` + NoRecursive bool `form:"noRecursive" url:"noRecursive" json:"noRecursive"` // 仅当isPrefix为true时有效，表示仅查询直接属于Prefix下的对象，对于更深的对象，返回它们的公共前缀 + MaxKeys int `form:"maxKeys" url:"maxKeys" json:"maxKeys"` + ContinuationToken string `form:"continuationToken" url:"continuationToken" json:"continuationToken"` // 用于分页，如果为空字符串，表示从头开始 } func (r *ObjectListByPath) MakeParam() *sdks.RequestParam { @@ -46,8 +47,10 @@ func (r *ObjectListByPath) MakeParam() *sdks.RequestParam { } type ObjectListByPathResp struct { - CommonPrefixes []string `json:"commonPrefixes"` // 仅在IsPrefix为true且NoRecursive为true时有效，包含更深层对象的shared prefix - Objects []cdssdk.Object `json:"objects"` // 如果IsPrefix为true且NoRecursive为false，则返回所有匹配的对象，否则只返回直接属于Prefix下的对象 + CommonPrefixes []string `json:"commonPrefixes"` // 仅在IsPrefix为true且NoRecursive为true时有效，包含更深层对象的shared prefix + Objects []cdssdk.Object `json:"objects"` // 如果IsPrefix为true且NoRecursive为false，则返回所有匹配的对象，否则只返回直接属于Prefix下的对象 + IsTruncated bool `json:"isTruncated"` // 是否还有更多对象 + NextContinuationToken string `json:"nextContinuationToken"` // 用于分页，如果IsTruncated为true，则下次请求的ContinuationToken为该值 } func (r *ObjectListByPathResp) ParseResponse(resp *http.Response) error { From 298256a902e6219b3672a518c85ca8199ad6bf93 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 27 Mar 2025 09:14:08 +0800 Subject: [PATCH 092/155] =?UTF-8?q?=E5=8A=A0=E9=94=81=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E5=B9=B6=E5=8F=91=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/io2/io.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/utils/io2/io.go b/utils/io2/io.go index 5107ae0..da57b92 100644 --- a/utils/io2/io.go +++ b/utils/io2/io.go @@ -1,6 +1,9 @@ package io2 -import "io" +import ( + "io" + "sync" +) type PromiseWriteCloser[T any] interface { io.Writer @@ -35,6 +38,7 @@ type readCloserHook struct { callback func(closer io.ReadCloser) once int isBefore bool // callback调用时机，true则在closer的Close之前调用 + lock *sync.Mutex } func (hook *readCloserHook) Read(buf []byte) (n int, err error) { @@ -42,6 +46,9 @@ func (hook *readCloserHook) Read(buf []byte) (n int, err error) { } func (hook *readCloserHook) Close() error { + hook.lock.Lock() + defer hook.lock.Unlock() + if hook.once == onceDone { return hook.readCloser.Close() } @@ -69,6 +76,7 @@ func BeforeReadClosing(closer io.ReadCloser, callback func(closer io.ReadCloser) callback: callback, once: onceDisabled, isBefore: true, + lock: &sync.Mutex{}, } } @@ -78,6 +86,7 @@ func AfterReadClosed(closer io.ReadCloser, callback func(closer io.ReadCloser)) callback: callback, once: onceDisabled, isBefore: false, + lock: &sync.Mutex{}, } } @@ -87,16 +96,22 @@ func AfterReadClosedOnce(closer io.ReadCloser, callback func(closer io.ReadClose callback: callback, once: onceEnabled, isBefore: false, + lock: &sync.Mutex{}, } } type afterEOF struct { inner io.ReadCloser callback func(str io.ReadCloser, err error) + lock *sync.Mutex } func (hook *afterEOF) Read(buf []byte) (n int, err error) { n, err = hook.inner.Read(buf) + + hook.lock.Lock() + defer hook.lock.Unlock() + if hook.callback != nil { if err == io.EOF { hook.callback(hook.inner, nil) @@ -111,6 +126,10 @@ func (hook *afterEOF) Read(buf []byte) (n int, err error) { func (hook *afterEOF) Close() error { err := hook.inner.Close() + + hook.lock.Lock() + defer hook.lock.Unlock() + if hook.callback != nil { hook.callback(hook.inner, io.ErrClosedPipe) hook.callback = nil @@ -122,6 +141,7 @@ func AfterEOF(str io.ReadCloser, callback func(str io.ReadCloser, err error)) io return &afterEOF{ inner: str, callback: callback, + lock: &sync.Mutex{}, } } From 2ce33f2288149d0a4b21aa85f6b3cc9016ca109d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 27 Mar 2025 10:31:35 +0800 Subject: [PATCH 093/155] =?UTF-8?q?=E4=BC=98=E5=8C=96hpc=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index f860bc1..df7f714 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -36,6 +36,7 @@ type CreateJobResp struct { type HPCJobInfo struct { JobDir string `json:"jobDir"` JobID string `json:"jobId"` + TaskID string `json:"taskId"` } func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { From 07fa8a6fcd60ad97102fa3ea9f8311946c79aebd Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 27 Mar 2025 11:11:19 +0800 Subject: [PATCH 094/155] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E6=94=AF=E6=8C=81=E9=A2=84=E7=AD=BE=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/presigned.go | 10 ++++++++++ sdks/storage/cdsapi/presigned_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go index b357e71..99bf8b2 100644 --- a/sdks/storage/cdsapi/presigned.go +++ b/sdks/storage/cdsapi/presigned.go @@ -23,6 +23,16 @@ func (c *Client) Presigned() *PresignedService { } } +const PresignedObjectListByPathPath = "/v1/presigned/object/listByPath" + +type PresignedObjectListByPath struct { + ObjectListByPath +} + +func (c *PresignedService) ObjectListByPath(req PresignedObjectListByPath, expireIn int) (string, error) { + return c.presign(req, PresignedObjectListByPathPath, http.MethodGet, expireIn) +} + const PresignedObjectDownloadByPathPath = "/v1/presigned/object/downloadByPath" type PresignedObjectDownloadByPath struct { diff --git a/sdks/storage/cdsapi/presigned_test.go b/sdks/storage/cdsapi/presigned_test.go index f4071c9..7379951 100644 --- a/sdks/storage/cdsapi/presigned_test.go +++ b/sdks/storage/cdsapi/presigned_test.go @@ -39,6 +39,31 @@ func Test_Presigned(t *testing.T) { }) } +func Test_PresignedObjectListByPath(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectListByPath(PresignedObjectListByPath{ + ObjectListByPath: ObjectListByPath{ + UserID: 1, + PackageID: 12, + Path: "a/", + IsPrefix: true, + NoRecursive: true, + MaxKeys: 10, + ContinuationToken: "123456", + }, + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + func Test_PresignedObjectDownloadByPath(t *testing.T) { cli := NewClient(&Config{ URL: "http://localhost:7890", From b936b4b0f1d4fe306ceaf8a6b20bbce475a66892 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 27 Mar 2025 15:08:25 +0800 Subject: [PATCH 095/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E8=AF=A6=E6=83=85=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 20 +++--- sdks/pcmscheduler/jobset.go | 6 +- sdks/pcmscheduler/models.go | 124 ++++++++---------------------------- 3 files changed, 38 insertions(+), 112 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index df7f714..2bc4c23 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -9,7 +9,7 @@ import ( "strings" ) -type CreateJobReq struct { +type CreateHPCJobReq struct { Name string `json:"name"` ClusterID schsdk.ClusterID `json:"clusterId"` Backend string `json:"backend"` @@ -19,14 +19,14 @@ type CreateJobReq struct { Parameters map[string]string `json:"parameters"` } -type HPCParameter struct { - JobName string `json:"jobName"` - Partition string `json:"partition"` - Ntasks string `json:"ntasks"` - Nodes string `json:"nodes"` - BamFile string `json:"bamFile"` - InputFile string `json:"inputFile"` -} +//type HPCParameter struct { +// JobName string `json:"jobName"` +// Partition string `json:"partition"` +// Ntasks string `json:"ntasks"` +// Nodes string `json:"nodes"` +// BamFile string `json:"bamFile"` +// InputFile string `json:"inputFile"` +//} type CreateJobResp struct { Backend string `json:"backend"` @@ -39,7 +39,7 @@ type HPCJobInfo struct { TaskID string `json:"taskId"` } -func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "/hpc/commitHpcTask") if err != nil { return nil, err diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index e6b8b41..87a6cfe 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -61,7 +61,7 @@ type CreateInferenceJobResp struct { TaskId string `json:"taskId"` } -type CreateJobReq struct { +type CreatePCMJobReq struct { Name string `json:"name"` Description string `json:"description"` JobResources schsdk.JobResources `json:"jobResources"` @@ -125,7 +125,7 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } -func (c *Client) CreateInferenceJob(req CreateJobReq, token string) (*CreateInferenceJobResp, error) { +func (c *Client) CreateInferenceJob(req CreatePCMJobReq, token string) (*CreateInferenceJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") if err != nil { return nil, err @@ -159,7 +159,7 @@ func (c *Client) CreateInferenceJob(req CreateJobReq, token string) (*CreateInfe } -func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreatePCMJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "schedule/createTask") if err != nil { return nil, err diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index b6895cd..c7b22bd 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -2,6 +2,7 @@ package sch import ( "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/sdks/hpc" schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/serder" @@ -239,122 +240,47 @@ type ResourceRange struct { Ids []string `json:"ids"` } -type QueryTaskResultReq struct { - TaskID string `json:"taskID" binding:"required"` - UserID cdssdk.UserID `json:"userID" binding:"required"` - PackageID cdssdk.PackageID `json:"packageID" binding:"required"` -} - -type PCMJobData struct { - ID string `json:"id"` - CreatedAt time.Time `json:"created_at"` - Param string `json:"param"` -} - type JobInfo struct { - Name string `json:"name"` - Description string `json:"description,optional"` - JobResources JobResources `json:"jobResources"` - DataDistributeData DataDistributeData `json:"dataDistributes"` - ResultFile ResultFile `json:"resultFile"` + TaskID string `json:"taskID"` + JobSubmitInfo JobSubmitInfo `json:"jobSubmitInfo"` + ResultFiles []ResultFile `json:"resultFiles"` } type ResultFile struct { - CommonPrefixes []interface{} `json:"commonPrefixes"` - Objects []Object `json:"objects"` -} - -// Object 表示对象结构体 -type Object struct { - ObjectID int `json:"objectID"` - PackageID int `json:"packageID"` - Path string `json:"path"` - Size string `json:"size"` - FileHash string `json:"fileHash"` - Redundancy Redundancy `json:"redundancy"` - CreateTime time.Time `json:"createTime"` - UpdateTime time.Time `json:"updateTime"` -} - -// Redundancy 表示冗余信息结构体 -type Redundancy struct { - Type string `json:"type"` - RepCount int `json:"repCount"` -} - -type DataDistributeData struct { - Dataset []DataItem `json:"dataset"` - Code []DataItem `json:"code"` - Image []DataItem `json:"image"` - Model []DataItem `json:"model"` -} - -// DataItem 定义 dataDistributes 中的数据项结构体 -type DataItem struct { - DataName string `json:"dataName"` - PackageID int `json:"packageID"` - PackageName string `json:"packageName"` - ResourceFile string `json:"resourceFile"` - Clusters []DataDetail `json:"clusters"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Objects []cdssdk.Object `json:"objects"` } -type JobResources struct { - ScheduleStrategy string `json:"scheduleStrategy"` - Clusters []*JobClusterInfo `json:"clusters"` +type JobSubmitInfo interface { + Noop() } -type JobClusterInfo struct { - ClusterID string `json:"clusterID"` - Resources []map[string]interface{} `json:"resources,optional"` - Runtime JobRuntimeInfo `json:"runtime,optional"` +type JobSubmitInfoBase struct { } -type JobRuntimeInfo struct { - Command string `json:"command,optional"` - Envs map[string]interface{} `json:"envs,optional"` - Params map[string]interface{} `json:"params,optional"` -} +var JobSubmitInfoTypeUnion = types.NewTypeUnion[JobSubmitInfo]( + (*PCMJobSubmitInfo)(nil), + (*HPCJobSubmitInfo)(nil), +) -// Engine 定义 engine 结构体 -type Engine struct { - EngineName string `json:"EngineName"` - EngineVersion string `json:"EngineVersion"` - ImageUrl string `json:"ImageUrl"` - InstallSysPackages bool `json:"InstallSysPackages"` -} +var _ = serder.UseTypeUnionInternallyTagged(&JobSubmitInfoTypeUnion, "type") -// bindingCluster 定义根结构体 -type BindingParam struct { - Name string `json:"name"` - Description string `json:"description"` - Engine Engine `json:"engine"` - CodeDir string `json:"codeDir"` - BootFile string `json:"bootFile"` - Branch string `json:"branch"` - Platform string `json:"platform"` - UserID int `json:"userID"` - PackageId int `json:"packageId"` - BucketID int `json:"bucketID"` -} +func (d *JobSubmitInfoBase) Noop() {} -// bindingCluster 定义根结构体 -type BindingCluster struct { - BindingId int `json:"binding_id"` - ClusterId string `json:"cluster_id"` - Status string `json:"status"` - Param string `json:"param"` +type PCMJobSubmitInfo struct { + serder.Metadata `union:"pcm"` + JobSubmitInfoBase + Type string `json:"type"` + Info CreatePCMJobReq `json:"info"` } -/*type TaskResult struct { - Name string `json:"name"` - Description string `json:"description"` - DataDistribute DataDistribute `json:"dataDistributes"` +type HPCJobSubmitInfo struct { + serder.Metadata `union:"hpc"` + JobSubmitInfoBase + Type string `json:"type"` + Info hpc.CreateHPCJobReq `json:"info"` } -type ClusterResources struct { - ClusterId string `json:"clusterId"` -}*/ - type Range struct { Min float64 `json:"min"` Max float64 `json:"max"` From 7abebedddd685b219cf10581efdfa805bd24db58 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 27 Mar 2025 17:03:15 +0800 Subject: [PATCH 096/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/logger/global_logger.go | 8 ++++++++ pkgs/trie/trie.go | 18 ++++++++++++------ sdks/storage/models.go | 16 +++++++++------- sdks/storage/utils.go | 5 +++++ utils/lo2/lo.go | 21 +++++++++++++++++++++ utils/math2/math.go | 12 ++++++++++++ 6 files changed, 67 insertions(+), 13 deletions(-) diff --git a/pkgs/logger/global_logger.go b/pkgs/logger/global_logger.go index 1a577cc..6d07e73 100644 --- a/pkgs/logger/global_logger.go +++ b/pkgs/logger/global_logger.go @@ -83,6 +83,14 @@ func Init(cfg *Config) error { // 下面是日志记录的方法，它们分别对应不同的日志级别和格式。 // 这些方法最终都会调用logrus对应的方法来记录日志。 +func Trace(args ...interface{}) { + logrus.Trace(args...) +} + +func Tracef(format string, args ...interface{}) { + logrus.Tracef(format, args...) +} + func Debug(args ...interface{}) { logrus.Debug(args...) } diff --git a/pkgs/trie/trie.go b/pkgs/trie/trie.go index 829a49f..4f5d2f9 100644 --- a/pkgs/trie/trie.go +++ b/pkgs/trie/trie.go @@ -102,10 +102,15 @@ func (n *Node[T]) RemoveSelf(cleanParent bool) { } // 修改时需要注意允许在visitorFn中删除当前节点 -func (n *Node[T]) Iterate(visitorFn func(word string, node *Node[T], isWordNode bool) VisitCtrl) { +func (n *Node[T]) Iterate(visitorFn func(path []string, node *Node[T], isWordNode bool) VisitCtrl) { + n.iterate([]string{}, visitorFn) +} + +func (n *Node[T]) iterate(path []string, visitorFn func(path []string, node *Node[T], isWordNode bool) VisitCtrl) { if n.WordNexts != nil { for word, node := range n.WordNexts { - ret := visitorFn(word, node, true) + newPath := append(path, word) + ret := visitorFn(newPath, node, true) if ret == VisitBreak { return } @@ -114,12 +119,13 @@ func (n *Node[T]) Iterate(visitorFn func(word string, node *Node[T], isWordNode continue } - node.Iterate(visitorFn) + node.iterate(newPath, visitorFn) } } if n.AnyNext != nil { - ret := visitorFn("", n.AnyNext, false) + newPath := append(path, "") + ret := visitorFn(newPath, n.AnyNext, false) if ret == VisitBreak { return } @@ -128,7 +134,7 @@ func (n *Node[T]) Iterate(visitorFn func(word string, node *Node[T], isWordNode return } - n.AnyNext.Iterate(visitorFn) + n.AnyNext.iterate(newPath, visitorFn) } } @@ -198,6 +204,6 @@ func (t *Trie[T]) CreateWords(words []string) *Node[T] { return ptr } -func (n *Trie[T]) Iterate(visitorFn func(word string, node *Node[T], isWordNode bool) VisitCtrl) { +func (n *Trie[T]) Iterate(visitorFn func(path []string, node *Node[T], isWordNode bool) VisitCtrl) { n.Root.Iterate(visitorFn) } diff --git a/sdks/storage/models.go b/sdks/storage/models.go index 293618a..ecf4867 100644 --- a/sdks/storage/models.go +++ b/sdks/storage/models.go @@ -257,10 +257,11 @@ const ( ) type Package struct { - PackageID PackageID `gorm:"column:PackageID; primaryKey; type:bigint; autoIncrement" json:"packageID"` - Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` - BucketID BucketID `gorm:"column:BucketID; type:bigint; not null" json:"bucketID"` - State string `gorm:"column:State; type:varchar(255); not null" json:"state"` + PackageID PackageID `gorm:"column:PackageID; primaryKey; type:bigint; autoIncrement" json:"packageID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + BucketID BucketID `gorm:"column:BucketID; type:bigint; not null" json:"bucketID"` + CreateTime time.Time `gorm:"column:CreateTime; type:datetime; not null" json:"createTime"` + State string `gorm:"column:State; type:varchar(255); not null" json:"state"` } func (Package) TableName() string { @@ -337,9 +338,10 @@ func (PinnedObject) TableName() string { } type Bucket struct { - BucketID BucketID `gorm:"column:BucketID; primaryKey; type:bigint; autoIncrement" json:"bucketID"` - Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` - CreatorID UserID `gorm:"column:CreatorID; type:bigint; not null" json:"creatorID"` + BucketID BucketID `gorm:"column:BucketID; primaryKey; type:bigint; autoIncrement" json:"bucketID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + CreateTime time.Time `gorm:"column:CreateTime; type:datetime; not null" json:"createTime"` + CreatorID UserID `gorm:"column:CreatorID; type:bigint; not null" json:"creatorID"` } func (Bucket) TableName() string { diff --git a/sdks/storage/utils.go b/sdks/storage/utils.go index 39268dd..94c6813 100644 --- a/sdks/storage/utils.go +++ b/sdks/storage/utils.go @@ -11,3 +11,8 @@ func JoinObjectPath(comps ...string) string { func SplitObjectPath(pat string) []string { return strings.Split(pat, ObjectPathSeparator) } + +func BaseName(pat string) string { + idx := strings.LastIndex(pat, ObjectPathSeparator) + return pat[idx+1:] +} diff --git a/utils/lo2/lo.go b/utils/lo2/lo.go index 930f6d3..41b794c 100644 --- a/utils/lo2/lo.go +++ b/utils/lo2/lo.go @@ -73,3 +73,24 @@ func Deref[T any](arr []*T) []T { return result } + +func AppendNew[T any](arr []T, items ...T) []T { + narr := make([]T, 0, len(arr)+len(items)) + narr = append(narr, arr...) + narr = append(narr, items...) + return narr +} + +func ArrayEquals[T comparable](arr1, arr2 []T) bool { + if len(arr1) != len(arr2) { + return false + } + + for i := 0; i < len(arr1); i++ { + if arr1[i] != arr2[i] { + return false + } + } + + return true +} diff --git a/utils/math2/math.go b/utils/math2/math.go index 4e6d268..d1bab7d 100644 --- a/utils/math2/math.go +++ b/utils/math2/math.go @@ -2,6 +2,18 @@ package math2 import "golang.org/x/exp/constraints" +func Sign[T constraints.Signed](v T) int { + if v > 0 { + return 1 + } + + if v < 0 { + return -1 + } + + return 0 +} + func Max[T constraints.Ordered](v1, v2 T) T { if v1 < v2 { return v2 From d66f32af17037f00c4e2540365bb8a392929eb1b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 31 Mar 2025 17:26:47 +0800 Subject: [PATCH 097/155] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E6=B5=81=E6=97=B6=E9=94=99=E8=AF=AF=E5=A4=84?= =?UTF-8?q?=E7=90=86EOF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/io2/range.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/utils/io2/range.go b/utils/io2/range.go index a4a0b35..46b4add 100644 --- a/utils/io2/range.go +++ b/utils/io2/range.go @@ -37,16 +37,22 @@ func (r *rng) Read(p []byte) (n int, err error) { need := math2.Min(*r.length, int64(len(p))) rd, err := r.inner.Read(p[:need]) - if err != nil { - r.err = err - return rd, io.EOF - } *r.length -= int64(rd) if *r.length == 0 { r.err = io.EOF } + if err == nil { + return rd, nil + } + + if err != io.EOF { + r.err = err + return rd, err + } + + r.err = io.EOF return rd, nil } From 3077125ab45dcd5717b2edbf679adc203243fd87 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 2 Apr 2025 10:27:00 +0800 Subject: [PATCH 098/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=92=8C=E5=A4=8D=E5=88=B6=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index a20cf30..991a3f6 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -125,6 +125,7 @@ type Package struct { Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` + BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` } type PackageDAO struct { @@ -139,6 +140,7 @@ type PackageDAO struct { UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` UploadPriority string `gorm:"column:upload_priority" json:"uploadPriority"` + Param string `gorm:"column:param" json:"param"` } type PackageCloneDAO struct { From e55bb85d3cfbd73a26c7a896f8dc10915bcc2759 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 2 Apr 2025 16:03:41 +0800 Subject: [PATCH 099/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=8A=E9=93=BE?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 5 ----- sdks/hpc/job.go | 1 + sdks/pcmscheduler/models.go | 2 +- sdks/scheduler/models.go | 1 + 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go index a7cb793..8d67862 100644 --- a/sdks/blockchain/blockchain.go +++ b/sdks/blockchain/blockchain.go @@ -22,11 +22,6 @@ func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { return err } - //token, err := c.getToken() - //if err != nil { - // return err - //} - header := make(map[string]string) header["Content-Type"] = http2.ContentTypeJSON header["Authorization"] = token diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 2bc4c23..c102fc6 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -11,6 +11,7 @@ import ( type CreateHPCJobReq struct { Name string `json:"name"` + Description string `json:"description"` ClusterID schsdk.ClusterID `json:"clusterId"` Backend string `json:"backend"` App string `json:"app"` diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index c7b22bd..4ba183c 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -442,7 +442,7 @@ type DatasetBinding struct { Category string `json:"category"` PackageID cdssdk.PackageID `json:"packageID"` RepositoryName string `json:"repositoryName"` - ConsumptionPoints float64 `json:"points"` + ConsumptionPoints int64 `json:"points"` } type ModelBinding struct { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index e676645..a2190dd 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -166,6 +166,7 @@ type HPCJobInfo struct { JobInfoBase Type string `json:"type"` Name string `json:"name"` + Description string `json:"description"` ClusterID ClusterID `json:"clusterID"` Backend string `json:"backend"` App string `json:"app"` From fd75ed60427ae44d597fcf289507c294aa391488 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 7 Apr 2025 11:26:27 +0800 Subject: [PATCH 100/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=B0=83=E5=BA=A6?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index a2190dd..57039b1 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -181,6 +181,12 @@ type HPCParameter struct { Ntasks string `json:"ntasks"` Nodes string `json:"nodes"` BamFile string `json:"bamFile"` + HashType string `json:"hashType"` + AttackMode string `json:"attackMode"` + HashInput string `json:"hashInput"` + Mask string `json:"mask"` + Dictionary string `json:"dictionary"` + Dictionary2 string `json:"dictionary2"` HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` } From 62607ad384eac9a2eff942a8009e05c1f8eac45d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 7 Apr 2025 17:11:03 +0800 Subject: [PATCH 101/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AE=9A=E7=89=88?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 4ba183c..f7062b2 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -72,6 +72,8 @@ const ( ID = "id" Startup = "startup" + + Schedule = "schedule" ) type TaskID int64 @@ -437,6 +439,7 @@ type DatasetBinding struct { DataBindingBase Type string `json:"type"` Name string `json:"name"` + OperateType string `json:"operateType"` ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` Description string `json:"description"` Category string `json:"category"` @@ -450,6 +453,7 @@ type ModelBinding struct { DataBindingBase Type string `json:"type"` Name string `json:"name"` + OperateType string `json:"operateType"` ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` Description string `json:"description"` Category string `json:"category"` @@ -465,6 +469,7 @@ type CodeBinding struct { DataBindingBase Type string `json:"type"` Name string `json:"name"` + OperateType string `json:"operateType"` ClusterID schsdk.ClusterID `json:"clusterID"` Description string `json:"description"` ImageID schsdk.ImageID `json:"imageID"` @@ -491,12 +496,13 @@ type CodeBinding struct { type ImageBinding struct { serder.Metadata `union:"image"` DataBindingBase - Type string `json:"type"` - ID int64 `json:"id"` - Name string `json:"name"` - IDType string `json:"idType"` - ImageID string `json:"imageID"` - ClusterID schsdk.ClusterID `json:"clusterID"` + Type string `json:"type"` + ID int64 `json:"id"` + OperateType string `json:"operateType"` + Name string `json:"name"` + IDType string `json:"idType"` + ImageID string `json:"imageID"` + ClusterID schsdk.ClusterID `json:"clusterID"` } type Image struct { From 9f4cfeed16a406677d91451b72de27cfbc456c8c Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 10 Apr 2025 11:04:59 +0800 Subject: [PATCH 102/155] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=AD=9B=E9=80=89=E6=9D=A1=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index f7062b2..dd717e2 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -415,6 +415,7 @@ type QueryData struct { CurrentPage int `json:"currentPage" binding:"required"` PageSize int `json:"pageSize" binding:"required"` OrderBy string `json:"orderBy" binding:"required"` + PackageName string `json:"packageName"` } type DataBinding interface { From 2f11a0a6202f66285b5f00c7bb9760c9b6575486 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 10 Apr 2025 16:15:40 +0800 Subject: [PATCH 103/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=9B=B4=E6=8E=A5?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E6=A8=A1=E5=9E=8B=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 3 +++ sdks/uploader/models.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index dd717e2..ff19a48 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -29,6 +29,9 @@ const ( MODEL = "model" RESULT = "result" + PackageTypeNormal = "normal" + PackageTypeNull = "null" + OrderByName = "name" OrderBySize = "size" OrderByTime = "time" diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 991a3f6..d152928 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -126,6 +126,7 @@ type Package struct { //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` + PackageType string `gorm:"column:package_type" json:"packageType"` } type PackageDAO struct { @@ -141,6 +142,7 @@ type PackageDAO struct { Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` UploadPriority string `gorm:"column:upload_priority" json:"uploadPriority"` Param string `gorm:"column:param" json:"param"` + PackageType string `gorm:"column:package_type" json:"packageType"` } type PackageCloneDAO struct { From 7cb79231aec30d4d758db0c0a4f9ef6807796f08 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 11 Apr 2025 10:10:25 +0800 Subject: [PATCH 104/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=88=9B=E5=BB=BApac?= =?UTF-8?q?kage=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 13 +++++++------ sdks/scheduler/models.go | 17 +++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index c102fc6..5fc8d08 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -10,12 +10,13 @@ import ( ) type CreateHPCJobReq struct { - Name string `json:"name"` - Description string `json:"description"` - ClusterID schsdk.ClusterID `json:"clusterId"` - Backend string `json:"backend"` - App string `json:"app"` - OperateType string `json:"operateType"` + Name string `json:"name"` + Description string `json:"description"` + ClusterID schsdk.ClusterID `json:"clusterId"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + ScriptContent string `json:"scriptContent"` //Parameters HPCParameter `json:"parameters"` Parameters map[string]string `json:"parameters"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 57039b1..136ded1 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -164,14 +164,15 @@ type ModelBinding struct { type HPCJobInfo struct { serder.Metadata `union:"HPC"` JobInfoBase - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - ClusterID ClusterID `json:"clusterID"` - Backend string `json:"backend"` - App string `json:"app"` - OperateType string `json:"operateType"` - Parameters HPCParameter `json:"parameters"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + ClusterID ClusterID `json:"clusterID"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + ScriptContent string `json:"scriptContent"` + Parameters HPCParameter `json:"parameters"` } type HPCParameter struct { From 101de1ad57cf6ac97baf8345d8204e6e22d7b30b Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 11 Apr 2025 15:00:53 +0800 Subject: [PATCH 105/155] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=A2=9E=E5=8A=A0=E7=AD=9B=E9=80=89=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index ff19a48..e5d04ba 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -69,7 +69,7 @@ const ( PlatformPCM = "PCM" PlatformPCMInference = "PCM_Inference" - PlatformHPC = "HPC" + PlatformHPC = "HPCSlurm" URL = "url" ID = "id" From 3323f54d0f2a04472dea59eff0e07cc843ec6071 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 15 Apr 2025 14:31:00 +0800 Subject: [PATCH 106/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E6=8F=90=E4=BA=A4=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/jobflow.go | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 sdks/scheduler/jobflow.go diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go new file mode 100644 index 0000000..7fa61b5 --- /dev/null +++ b/sdks/scheduler/jobflow.go @@ -0,0 +1,66 @@ +package schsdk + +import "time" + +type FlowData struct { + Nodes []Node `json:"nodes"` + Edges []Edge `json:"edges"` +} + +type Node struct { + ID string `json:"id"` + Type string `json:"type"` + X int `json:"x"` + Y int `json:"y"` + Properties JobInfo `json:"properties"` + Text *Text `json:"text,omitempty"` // 有些节点没有 text 字段 +} + +type Edge struct { + ID string `json:"id"` + Type string `json:"type"` + Properties interface{} `json:"properties"` // 为空对象 {} + SourceNodeID string `json:"sourceNodeId"` + TargetNodeID string `json:"targetNodeId"` + SourceAnchorID string `json:"sourceAnchorId"` + TargetAnchorID string `json:"targetAnchorId"` + StartPoint Point `json:"startPoint"` + EndPoint Point `json:"endPoint"` + PointsList []Point `json:"pointsList"` + Text *Text `json:"text,omitempty"` // 有些 edge 有文字标签 +} + +type Point struct { + X int `json:"x"` + Y int `json:"y"` +} + +type Text struct { + X int `json:"x"` + Y int `json:"y"` + Value string `json:"value"` +} + +type JobFlowDAO struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID int64 `gorm:"column:user_id;not null" json:"UserID"` + Name string `gorm:"column:name;size:255;not null" json:"Name"` + Description string `gorm:"column:description;size:255" json:"Description"` + Content string `gorm:"column:content;type:text" json:"Content"` + Status string `gorm:"column:status;type:enum('pending','running','fail','success');default:'pending'" json:"Status"` + JobType string `gorm:"column:job_type;size:255" json:"JobType"` + UpdatedAt time.Time `gorm:"column:update_at;autoUpdateTime" json:"UpdatedAt"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"CreatedAt"` +} + +type JobFlow struct { + ID int64 `json:"ID"` + UserID int64 `json:"UserID"` + Name string `json:"Name"` + Description string `json:"Description"` + Content FlowData `json:"Content"` + Status string `json:"Status"` + JobType string `json:"JobType"` + UpdatedAt time.Time `json:"UpdatedAt"` + CreatedAt time.Time `json:"CreatedAt"` +} From f8a5e7c84c272b80b70deeea1e2f5b1881b0cda4 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 16 Apr 2025 11:16:51 +0800 Subject: [PATCH 107/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=AE=A1=E7=90=86=E7=9B=B8=E5=85=B3=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + sdks/scheduler/jobflow.go | 41 ++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index e5d04ba..9cdab8e 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -56,6 +56,7 @@ const ( FailedStatus = "failed" SuccessStatus = "success" UploadingStatus = "uploading" + RunningStatus = "running" Query = "query" Delete = "delete" diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index 7fa61b5..d6cf5fb 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -1,6 +1,9 @@ package schsdk -import "time" +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) type FlowData struct { Nodes []Node `json:"nodes"` @@ -42,25 +45,25 @@ type Text struct { } type JobFlowDAO struct { - ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` - UserID int64 `gorm:"column:user_id;not null" json:"UserID"` - Name string `gorm:"column:name;size:255;not null" json:"Name"` - Description string `gorm:"column:description;size:255" json:"Description"` - Content string `gorm:"column:content;type:text" json:"Content"` - Status string `gorm:"column:status;type:enum('pending','running','fail','success');default:'pending'" json:"Status"` - JobType string `gorm:"column:job_type;size:255" json:"JobType"` - UpdatedAt time.Time `gorm:"column:update_at;autoUpdateTime" json:"UpdatedAt"` - CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"CreatedAt"` + ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"id"` + UserID cdssdk.UserID `gorm:"column:user_id;not null" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content string `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('pending','running','failed','success');default:'pending'" json:"status"` + JobType string `gorm:"column:job_type;size:255" json:"jobType"` + UpdatedAt time.Time `gorm:"column:update_at;autoUpdateTime" json:"updatedAt"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` } type JobFlow struct { - ID int64 `json:"ID"` - UserID int64 `json:"UserID"` - Name string `json:"Name"` - Description string `json:"Description"` - Content FlowData `json:"Content"` - Status string `json:"Status"` - JobType string `json:"JobType"` - UpdatedAt time.Time `json:"UpdatedAt"` - CreatedAt time.Time `json:"CreatedAt"` + ID int64 `json:"id"` + UserID cdssdk.UserID `json:"userID"` + Name string `json:"name"` + Description string `json:"description"` + Content FlowData `json:"content"` + Status string `json:"status"` + JobType string `json:"jobType"` + UpdatedAt time.Time `json:"updatedAt"` + CreatedAt time.Time `json:"createdAt"` } From d33a7d71109321c6ac9d7f69cb50ed9f19cf339b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 16 Apr 2025 17:35:23 +0800 Subject: [PATCH 108/155] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=9B=E5=A2=9E=E5=8A=A0trace=E7=BA=A7=E5=88=AB=E6=97=A5?= =?UTF-8?q?=E5=BF=97=EF=BC=9B=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6=E5=8F=8D?= =?UTF-8?q?=E5=BA=8F=E5=88=97=E5=8C=96=E6=94=AF=E6=8C=81time.Duration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/ioswitch/plan/ops/driver.go | 4 +- pkgs/logger/logger.go | 3 ++ pkgs/logger/logrus_logger.go | 7 +++ utils/config/config.go | 13 ++++- utils/serder/json/exts/exts_test.go | 35 +++++++++++++ utils/serder/json/exts/time.go | 76 +++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 utils/serder/json/exts/exts_test.go create mode 100644 utils/serder/json/exts/time.go diff --git a/pkgs/ioswitch/plan/ops/driver.go b/pkgs/ioswitch/plan/ops/driver.go index 66f4395..3a50232 100644 --- a/pkgs/ioswitch/plan/ops/driver.go +++ b/pkgs/ioswitch/plan/ops/driver.go @@ -58,8 +58,8 @@ func (t *ToDriverNode) SetInput(v *dag.StreamVar) { v.To(t, 0) } -func (t *ToDriverNode) Input() dag.StreamOutputSlot { - return dag.StreamOutputSlot{ +func (t *ToDriverNode) Input() dag.StreamInputSlot { + return dag.StreamInputSlot{ Node: t, Index: 0, } diff --git a/pkgs/logger/logger.go b/pkgs/logger/logger.go index 3e0a52b..ad67986 100644 --- a/pkgs/logger/logger.go +++ b/pkgs/logger/logger.go @@ -30,6 +30,9 @@ var loggerLevels = map[string]logrus.Level{ } type Logger interface { + Trace(args ...interface{}) + Tracef(format string, args ...interface{}) + Debug(args ...interface{}) Debugf(format string, args ...interface{}) diff --git a/pkgs/logger/logrus_logger.go b/pkgs/logger/logrus_logger.go index a69eda3..16ba253 100644 --- a/pkgs/logger/logrus_logger.go +++ b/pkgs/logger/logrus_logger.go @@ -10,6 +10,13 @@ type logrusLogger struct { entry *logrus.Entry } +func (l *logrusLogger) Trace(args ...interface{}) { + l.entry.Trace(args...) +} +func (l *logrusLogger) Tracef(format string, args ...interface{}) { + l.entry.Tracef(format, args...) +} + func (l *logrusLogger) Debug(args ...interface{}) { l.entry.Debug(args...) } diff --git a/utils/config/config.go b/utils/config/config.go index d0f930b..ed98270 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -1,14 +1,23 @@ package config import ( - "encoding/json" "fmt" "os" "path/filepath" "github.com/imdario/mergo" + "gitlink.org.cn/cloudream/common/utils/serder/json" + "gitlink.org.cn/cloudream/common/utils/serder/json/exts" ) +var serder json.Serder + +func init() { + cfg := json.New() + cfg = cfg.UseExtension(exts.NewDuration()) + serder = cfg.Build() +} + // Load 从本地文件读取配置，加载配置文件 func Load(filePath string, cfg interface{}) error { fileData, err := os.ReadFile(filePath) @@ -17,7 +26,7 @@ func Load(filePath string, cfg interface{}) error { } // json.Unmarshal用于将JSON解码成结构体 - return json.Unmarshal(fileData, cfg) + return serder.Decode(fileData, cfg) } // DefaultLoad 默认的加载配置的方式： diff --git a/utils/serder/json/exts/exts_test.go b/utils/serder/json/exts/exts_test.go new file mode 100644 index 0000000..348c1e2 --- /dev/null +++ b/utils/serder/json/exts/exts_test.go @@ -0,0 +1,35 @@ +package exts + +import ( + "testing" + "time" + + jsoniter "github.com/json-iterator/go" + . "github.com/smartystreets/goconvey/convey" +) + +func Test_Duration(t *testing.T) { + c := jsoniter.Config{} + api := c.Froze() + api.RegisterExtension(&DurationExt{}) + + Convey("Duration", t, func() { + type Test struct { + D1 time.Duration `json:"d1"` + D2 time.Duration `json:"d2"` + D3 *time.Duration `json:"d3"` + D4 *time.Duration `json:"d4"` + D5 *time.Duration `json:"d5"` + } + + var test Test + err := api.Unmarshal([]byte(`{"d1":"1h", "d2": null, "d3": "2h", "d4": null}`), &test) + So(err, ShouldBeNil) + So(test.D1, ShouldEqual, 1*time.Hour) + So(test.D2, ShouldEqual, 0) + So(*test.D3, ShouldEqual, 2*time.Hour) + So(test.D4, ShouldEqual, (*time.Duration)(nil)) + So(test.D5, ShouldEqual, (*time.Duration)(nil)) + }) + +} diff --git a/utils/serder/json/exts/time.go b/utils/serder/json/exts/time.go new file mode 100644 index 0000000..03ffb81 --- /dev/null +++ b/utils/serder/json/exts/time.go @@ -0,0 +1,76 @@ +package exts + +import ( + "fmt" + "time" + "unsafe" + + jsoniter "github.com/json-iterator/go" + "github.com/modern-go/reflect2" +) + +type DurationExt struct { + jsoniter.DummyExtension +} + +func NewDuration() *DurationExt { + return &DurationExt{} +} + +func (e *DurationExt) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { + if typ == reflect2.TypeOf(time.Duration(0)) { + return &DurationDecoder{ + isPointer: false, + } + } + + if typ == reflect2.TypeOf((*time.Duration)(nil)) { + return &DurationDecoder{ + isPointer: true, + } + } + + return nil +} + +type DurationDecoder struct { + isPointer bool +} + +func (e *DurationDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + nextType := iter.WhatIsNext() + if nextType == jsoniter.StringValue { + str := iter.ReadString() + dt, err := time.ParseDuration(str) + if err != nil { + iter.Error = err + return + } + + if e.isPointer { + p := (**time.Duration)(ptr) + *p = &dt + } else { + p := (*time.Duration)(ptr) + *p = dt + } + + } else if nextType == jsoniter.NilValue { + iter.ReadNil() + + if e.isPointer { + p := (**time.Duration)(ptr) + *p = nil + } else { + p := (*time.Duration)(ptr) + *p = time.Duration(0) + } + + } else { + iter.Error = fmt.Errorf("unsupported type %v", nextType) + return + } + +} + +var _ jsoniter.Extension = &DurationExt{} From d8d739b1be137915d878ee69b66c7fd60a0ec97d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 17 Apr 2025 09:56:03 +0800 Subject: [PATCH 109/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=B6=85=E7=AE=97?= =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 136ded1..1c00a7f 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -192,8 +192,9 @@ type HPCParameter struct { } type HPCBindingFile struct { - ParamName string `json:"paramName"` - BindingID int64 `json:"bindingID"` + ParamName string `json:"paramName"` + ObjectID cdssdk.ObjectID `json:"objectID"` + PackageID cdssdk.PackageID `json:"packageID"` } type JobResources struct { From 63b6ead6667fd95c03c7c5d7af9113bf0fa17e53 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 17 Apr 2025 15:51:09 +0800 Subject: [PATCH 110/155] =?UTF-8?q?=E8=B0=83=E6=95=B4HPC=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 43 ++++++++++++++++++++++++++++++++++- sdks/scheduler/models.go | 6 ++--- utils/http2/http.go | 21 +++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go index 8d67862..5817b54 100644 --- a/sdks/blockchain/blockchain.go +++ b/sdks/blockchain/blockchain.go @@ -14,6 +14,7 @@ type InvokeReq struct { MemberName string `json:"memberName"` Type string `json:"type"` Args []string `json:"args"` + Amount int64 `json:"amount"` } func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { @@ -25,7 +26,6 @@ func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { header := make(map[string]string) header["Content-Type"] = http2.ContentTypeJSON header["Authorization"] = token - println("token: " + token) resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, @@ -50,6 +50,47 @@ func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { return nil } +type UnPledgePointsReq struct { + PrivateKey string `form:"privateKey"` + BusinessCode string `form:"businessCode"` +} + +func (c *Client) UnPledgePoints(req UnPledgePointsReq, token string) error { + targetUrl, err := url.JoinPath(c.baseURL, "/jcc-bcos/points/unPledgePoints") + if err != nil { + return err + } + + header := make(map[string]string) + header["Content-Type"] = http2.ContentTypeJSON + header["Authorization"] = token + + resp, err := http2.PutJSON(targetUrl, http2.RequestParam{ + Query: req, + Header: header, + }) + if err != nil { + println(err) + return err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == 200 { + return nil + } + + return codeResp.ToError() + } + + return nil +} + type TokenReq struct { Username string `json:"username"` Password string `json:"password"` diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 1c00a7f..cfdd689 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -192,9 +192,9 @@ type HPCParameter struct { } type HPCBindingFile struct { - ParamName string `json:"paramName"` - ObjectID cdssdk.ObjectID `json:"objectID"` - PackageID cdssdk.PackageID `json:"packageID"` + ParamName string `json:"paramName"` + ObjectID cdssdk.ObjectID `json:"objectID"` + //PackageID cdssdk.PackageID `json:"packageID"` } type JobResources struct { diff --git a/utils/http2/http.go b/utils/http2/http.go index ab67e21..f57b06f 100644 --- a/utils/http2/http.go +++ b/utils/http2/http.go @@ -119,6 +119,27 @@ func PostJSON(url string, param RequestParam) (*http.Response, error) { return defaultClient.Do(req) } +func PutJSON(url string, param RequestParam) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPut, url, nil) + if err != nil { + return nil, err + } + + if err = prepareQuery(req, param.Query); err != nil { + return nil, err + } + + if err = prepareHeader(req, param.Header); err != nil { + return nil, err + } + + if err = prepareJSONBody(req, param.Body); err != nil { + return nil, err + } + + return defaultClient.Do(req) +} + func PostForm(url string, param RequestParam) (*http.Response, error) { req, err := http.NewRequest(http.MethodPost, url, nil) if err != nil { From b8efb988a75e3fc00b4fc49d18f4573473dba120 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 17 Apr 2025 16:12:55 +0800 Subject: [PATCH 111/155] =?UTF-8?q?=E8=B0=83=E6=95=B4HPC=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=8F=82=E6=95=B02?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index cfdd689..e4e3f38 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -194,7 +194,7 @@ type HPCParameter struct { type HPCBindingFile struct { ParamName string `json:"paramName"` ObjectID cdssdk.ObjectID `json:"objectID"` - //PackageID cdssdk.PackageID `json:"packageID"` + Path string `json:"path"` } type JobResources struct { From 5685f35153601cc2c126f0806624bed4f41577b4 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 18 Apr 2025 09:09:10 +0800 Subject: [PATCH 112/155] =?UTF-8?q?=E7=A7=AF=E5=88=86=E8=A7=A3=E9=99=A4?= =?UTF-8?q?=E8=B4=A8=E6=8A=BC=E4=B8=8A=E9=93=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/job.go | 1 + sdks/pcmscheduler/models.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/sdks/pcmscheduler/job.go b/sdks/pcmscheduler/job.go index fd1997b..3a5d291 100644 --- a/sdks/pcmscheduler/job.go +++ b/sdks/pcmscheduler/job.go @@ -13,6 +13,7 @@ type PCMJob struct { JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"` LocalJobID string `gorm:"column:local_job_id" json:"localJobID"` Param string `gorm:"column:param" json:"param"` + Token string `gorm:"column:token" json:"token"` CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 9cdab8e..32fa4f2 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -78,6 +78,8 @@ const ( Startup = "startup" Schedule = "schedule" + + BlockChainJobCreatePrefix = "job_create_" ) type TaskID int64 From a7dc1ec0bc5f6bc203701396925c8e2b63c32cb3 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 18 Apr 2025 10:30:40 +0800 Subject: [PATCH 113/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index e4e3f38..9188981 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -192,9 +192,38 @@ type HPCParameter struct { } type HPCBindingFile struct { - ParamName string `json:"paramName"` - ObjectID cdssdk.ObjectID `json:"objectID"` - Path string `json:"path"` + ParamName string `json:"paramName"` + Resource HPCFile `json:"resource"` +} + +type HPCFile interface { + Noop() +} + +var HPCFileTypeUnion = types.NewTypeUnion[HPCFile]( + (*HPCObject)(nil), + (*HPCPath)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&HPCFileTypeUnion, "type") + +type HPCFileBase struct{} + +func (d *HPCFileBase) Noop() {} + +type HPCObject struct { + serder.Metadata `union:"object"` + HPCFileBase + Type string `json:"type"` + ObjectID cdssdk.ObjectID `json:"objectID"` +} + +type HPCPath struct { + serder.Metadata `union:"path"` + HPCFileBase + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` + Path string `json:"path"` } type JobResources struct { From 4b575059a0e3a1083d6cb593b5561348710006af Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 21 Apr 2025 10:32:15 +0800 Subject: [PATCH 114/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8A=E9=93=BE?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 10 ++++++++++ sdks/scheduler/models.go | 28 ++++++++++++---------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 5fc8d08..f7f6710 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -5,6 +5,7 @@ import ( schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" + "io" "net/url" "strings" ) @@ -57,6 +58,15 @@ func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, e return nil, err } + // 打印resp.Body内容 + body, err := io.ReadAll(resp.Body) + if err != nil { + println("Error reading response body:", err) + } + + // Print the response body as a string + println("Response Body:", string(body)) + contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { var codeResp respons2[CreateJobResp] diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9188981..c7fa3fc 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -51,7 +51,6 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string GetTargetLocalJobIDs() []string - RemoveTargetLocalJobID(targetID string) } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -70,8 +69,14 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` - TargetLocalJobIDs []string `json:"targetLocalJobIDs"` + LocalJobID string `json:"localJobID"` + //TargetLocalJobIDs []string `json:"targetLocalJobIDs"` + TargetJobInfos []TargetJobInfo `json:"targetJobInfos"` +} + +type TargetJobInfo struct { + TargetJobID string `json:"targetJobID"` + InputParams map[string]string `json:"inputParams"` } func (i *JobInfoBase) GetLocalJobID() string { @@ -79,20 +84,11 @@ func (i *JobInfoBase) GetLocalJobID() string { } func (i *JobInfoBase) GetTargetLocalJobIDs() []string { - return i.TargetLocalJobIDs -} - -func (i *JobInfoBase) GetTargetLocalJobIDs2() []string { - return i.TargetLocalJobIDs -} - -func (i *JobInfoBase) RemoveTargetLocalJobID(targetID string) { - // 从i.TargetLocalJobIDs中删除id - for j, id := range i.TargetLocalJobIDs { - if targetID == id { - i.TargetLocalJobIDs = append(i.TargetLocalJobIDs[:j], i.TargetLocalJobIDs[j+1:]...) - } + var IDs []string + for _, v := range i.TargetJobInfos { + IDs = append(IDs, v.TargetJobID) } + return IDs } type NormalJobInfo struct { From 672cdcf4609f079f9e6a3bbf5c77236dd5ce21e4 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 21 Apr 2025 10:54:15 +0800 Subject: [PATCH 115/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8A=E9=93=BE?= =?UTF-8?q?=E4=BB=A3=E7=A0=812?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index f7f6710..18338bc 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -5,7 +5,6 @@ import ( schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" - "io" "net/url" "strings" ) @@ -59,13 +58,12 @@ func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, e } // 打印resp.Body内容 - body, err := io.ReadAll(resp.Body) - if err != nil { - println("Error reading response body:", err) - } - - // Print the response body as a string - println("Response Body:", string(body)) + //body, err := io.ReadAll(resp.Body) + //if err != nil { + // println("Error reading response body:", err) + //} + // + //println("Response Body:", string(body)) contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { From f96bed7ad7562684a2bc496978dbbbec14474a37 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 21 Apr 2025 16:32:38 +0800 Subject: [PATCH 116/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E5=80=BC=E4=BC=A0=E9=80=92=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index c7fa3fc..abe5eaf 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -51,6 +51,7 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string GetTargetLocalJobIDs() []string + GetTargetInputParams(targetID string) map[string]string } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -71,7 +72,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { LocalJobID string `json:"localJobID"` //TargetLocalJobIDs []string `json:"targetLocalJobIDs"` - TargetJobInfos []TargetJobInfo `json:"targetJobInfos"` + TargetJob []TargetJobInfo `json:"targetJob"` } type TargetJobInfo struct { @@ -83,9 +84,18 @@ func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } +func (i *JobInfoBase) GetTargetInputParams(targetID string) map[string]string { + for _, v := range i.TargetJob { + if v.TargetJobID == targetID { + return v.InputParams + } + } + return nil +} + func (i *JobInfoBase) GetTargetLocalJobIDs() []string { var IDs []string - for _, v := range i.TargetJobInfos { + for _, v := range i.TargetJob { IDs = append(IDs, v.TargetJobID) } return IDs @@ -357,10 +367,10 @@ type DataPreprocessJobInfo struct { type DataReturnJobInfo struct { serder.Metadata `union:"DataReturn"` JobInfoBase - Type string `json:"type"` - BucketID cdssdk.BucketID `json:"bucketID"` - BindingType string `json:"bindingType"` - TargetLocalJobID string `json:"targetLocalJobID"` + Type string `json:"type"` + BucketID cdssdk.BucketID `json:"bucketID"` + //BindingType string `json:"bindingType"` + TargetLocalJobID string `json:"targetLocalJobID"` } // MultiInstanceJobInfo 多实例(推理任务) From ac8bc4c1e3f899831e7c00bab2d0e765537b4b79 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 22 Apr 2025 10:29:43 +0800 Subject: [PATCH 117/155] =?UTF-8?q?=E4=BC=98=E5=8C=96WalkEnd=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E8=BF=94=E5=9B=9E=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/trie/trie.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/trie/trie.go b/pkgs/trie/trie.go index 4f5d2f9..18e340c 100644 --- a/pkgs/trie/trie.go +++ b/pkgs/trie/trie.go @@ -166,10 +166,11 @@ func (t *Trie[T]) WalkEnd(words []string) (*Node[T], bool) { ptr := &t.Root for _, word := range words { - ptr = ptr.WalkNext(word) - if ptr == nil { - return nil, false + np := ptr.WalkNext(word) + if np == nil { + return ptr, false } + ptr = np } return ptr, true From 6d0275324b8611cec87400724d179a28bac4d8e3 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 22 Apr 2025 15:28:30 +0800 Subject: [PATCH 118/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E8=AF=A6=E6=83=85=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index d152928..35ebf1d 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -112,30 +112,27 @@ func (Cluster) TableName() string { } type Package struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:create_time" json:"createTime"` - Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` - UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` - Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` - //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 - UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` - BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` - PackageType string `gorm:"column:package_type" json:"packageType"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` + Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` + UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` + Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` + UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` + BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` + PackageType string `gorm:"column:package_type" json:"packageType"` } type PackageDAO struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:create_time" json:"createTime"` UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 From d612d333d249199b992cf97bd390e0199a1ccadd Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 23 Apr 2025 09:19:58 +0800 Subject: [PATCH 119/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E6=99=BA=E7=AE=97=E4=BB=BB=E5=8A=A1=E7=9A=84output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 18338bc..8c9acad 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -62,7 +62,6 @@ func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, e //if err != nil { // println("Error reading response body:", err) //} - // //println("Response Body:", string(body)) contType := resp.Header.Get("Content-Type") From b97e7fb2c1097048bc05015b8db8344b4d93693d Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 23 Apr 2025 10:22:08 +0800 Subject: [PATCH 120/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=A4=96=E9=83=A8=E5=AD=98=E5=82=A8=E9=87=8C=E7=9A=84=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E7=9A=84=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/storage.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index d275432..68c1f00 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -79,3 +79,25 @@ func (r *StorageGetResp) ParseResponse(resp *http.Response) error { func (c *Client) StorageGet(req StorageGet) (*StorageGetResp, error) { return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageGetResp{}) } + +const StorageDeleteFilesPath = "/v1/storage/deleteFiles" + +type StorageDeleteFiles struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + StorageID cdssdk.StorageID `json:"storageID" binding:"required"` + Pathes []string `json:"pathes"` +} + +func (r *StorageDeleteFiles) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, StorageDeleteFilesPath, r) +} + +type StorageDeleteFilesResp struct{} + +func (r *StorageDeleteFilesResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *Client) StorageDeleteFiles(req StorageDeleteFiles) (*StorageDeleteFilesResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageDeleteFilesResp{}) +} From 44a51fb2d9adb826b500a0978cd07c6a167463eb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 23 Apr 2025 10:23:18 +0800 Subject: [PATCH 121/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=99=BA=E7=AE=97?= =?UTF-8?q?=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 7 ++++--- sdks/scheduler/models.go | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 32fa4f2..133b50e 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -68,9 +68,10 @@ const ( PlatformOpenI = "OpenI" PlatformModelArts = "ModelArts" - PlatformPCM = "PCM" - PlatformPCMInference = "PCM_Inference" - PlatformHPC = "HPCSlurm" + PlatformAI = "AI" // 智算 + PlatformCloud = "CLOUD" // 云算 + PlatformCloudInference = "PCM_Inference" + PlatformHPC = "HPCSlurm" //超算 URL = "url" ID = "id" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index abe5eaf..cd182d7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -62,7 +62,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*UpdateMultiInstanceJobInfo)(nil), (*FinetuningJobInfo)(nil), (*DataPreprocessJobInfo)(nil), - (*PCMJobInfo)(nil), + (*AIJobInfo)(nil), (*HPCJobInfo)(nil), (*BindingJobInfo)(nil), (*PCMInferenceJobInfo)(nil), @@ -122,8 +122,8 @@ type PCMInferenceJobInfo struct { JobResources JobResources `json:"jobResources"` } -type PCMJobInfo struct { - serder.Metadata `union:"PCM"` +type AIJobInfo struct { + serder.Metadata `union:"AI"` JobInfoBase Type string `json:"type"` Name string `json:"name"` From b68677f3d8453341b960e77b6a4abe7d4e136e5d Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 23 Apr 2025 15:37:02 +0800 Subject: [PATCH 122/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=8E=8B=E7=BC=A9?= =?UTF-8?q?=E4=B8=8B=E8=BD=BDPackage=E7=9A=84=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/package.go | 70 ++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index 6c1d2c3..a3223c0 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -1,10 +1,17 @@ package cdsapi import ( + "context" "fmt" + "io" + "mime" "net/http" "net/url" + "strings" + "time" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/sdks" @@ -146,6 +153,69 @@ func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadRe return nil, codeResp.ToError() } +const PackageDownloadPath = "/v1/package/download" + +type PackageDownload struct { + UserID cdssdk.UserID `url:"userID" form:"userID" binding:"required"` + PackageID cdssdk.PackageID `url:"packageID" form:"packageID" binding:"required"` +} + +func (r *PackageDownload) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodGet, PackageDownloadPath, r) +} + +type DownloadingPackage struct { + Name string + File io.ReadCloser +} + +func (c *PackageService) Download(req PackageDownload) (*DownloadingPackage, error) { + httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL) + if err != nil { + return nil, err + } + + if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[any] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + return nil, codeResp.ToError() + } + + _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) + if err != nil { + return nil, fmt.Errorf("parsing content disposition: %w", err) + } + + return &DownloadingPackage{ + Name: params["filename"], + File: resp.Body, + }, nil +} + const PackageDeletePath = "/package/delete" type PackageDelete struct { From d250d681c226bd709129d3d557e11afb8c1e60b0 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 23 Apr 2025 17:06:10 +0800 Subject: [PATCH 123/155] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81X,Y=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/jobflow.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index d6cf5fb..5219b80 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -13,8 +13,8 @@ type FlowData struct { type Node struct { ID string `json:"id"` Type string `json:"type"` - X int `json:"x"` - Y int `json:"y"` + X float64 `json:"x"` + Y float64 `json:"y"` Properties JobInfo `json:"properties"` Text *Text `json:"text,omitempty"` // 有些节点没有 text 字段 } @@ -34,14 +34,14 @@ type Edge struct { } type Point struct { - X int `json:"x"` - Y int `json:"y"` + X float64 `json:"x"` + Y float64 `json:"y"` } type Text struct { - X int `json:"x"` - Y int `json:"y"` - Value string `json:"value"` + X float64 `json:"x"` + Y float64 `json:"y"` + Value string `json:"value"` } type JobFlowDAO struct { From 20c97f8e1a3c15c9bebca4b14188509ec9a9c47d Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 24 Apr 2025 09:28:25 +0800 Subject: [PATCH 124/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=AE=A1=E7=AE=97Pac?= =?UTF-8?q?kage=E5=93=88=E5=B8=8C=E7=9A=84=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/package.go | 23 +++++++++++++++++++++++ sdks/storage/models.go | 5 +++++ 2 files changed, 28 insertions(+) diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index a3223c0..d406fe4 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -307,3 +307,26 @@ func (r *PackageGetCachedStoragesResp) ParseResponse(resp *http.Response) error func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetCachedStoragesResp{}) } + +const PackageCalcHashPath = "/v1/package/calcHash" + +type PackageCalcHash struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` +} + +func (r *PackageCalcHash) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageCalcHashPath, r) +} + +type PackageCalcHashResp struct { + cdssdk.PackageHash +} + +func (r *PackageCalcHashResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *PackageService) CalcHash(req PackageCalcHash) (*PackageCalcHashResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageCalcHashResp{}) +} diff --git a/sdks/storage/models.go b/sdks/storage/models.go index ecf4867..9a53072 100644 --- a/sdks/storage/models.go +++ b/sdks/storage/models.go @@ -385,3 +385,8 @@ type CodeError struct { func (e *CodeError) Error() string { return fmt.Sprintf("code: %s, message: %s", e.Code, e.Message) } + +type PackageHash struct { + // 16进制字符串格式的sha256哈希值 + Hash string `json:"hash"` +} From ab3814845650281c64057e26f50e6de6be4379a4 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 24 Apr 2025 11:05:32 +0800 Subject: [PATCH 125/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=93=E5=8D=B0?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=9A=84=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/ioswitch/dag/graph.go | 99 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/pkgs/ioswitch/dag/graph.go b/pkgs/ioswitch/dag/graph.go index 1a4e0f1..99ed675 100644 --- a/pkgs/ioswitch/dag/graph.go +++ b/pkgs/ioswitch/dag/graph.go @@ -1,7 +1,12 @@ package dag import ( + "fmt" + "reflect" + "strings" + "gitlink.org.cn/cloudream/common/utils/lo2" + "gitlink.org.cn/cloudream/common/utils/reflect2" ) type Graph struct { @@ -55,6 +60,100 @@ func (g *Graph) NewValueVar() *ValueVar { return &ValueVar{} } +func (g *Graph) Dump() string { + nodeIDs := make(map[Node]int) + for i, node := range g.Nodes { + nodeIDs[node] = i + } + + var sb strings.Builder + for _, node := range g.Nodes { + id, ok := nodeIDs[node] + if !ok { + id = len(nodeIDs) + nodeIDs[node] = id + } + sb.WriteString(fmt.Sprintf("[%v]%v\n", id, nodeTypeName(node))) + if node.InputStreams().Len() > 0 { + sb.WriteString("SIn: ") + for i, in := range node.InputStreams().Slots { + if i > 0 { + sb.WriteString(", ") + } + + if in == nil { + sb.WriteString("?") + } else { + sb.WriteString(fmt.Sprintf("%v", nodeIDs[in.Src])) + } + } + sb.WriteString("\n") + } + if node.OutputStreams().Len() > 0 { + sb.WriteString("SOut: ") + for i, out := range node.OutputStreams().Slots { + if i > 0 { + sb.WriteString(", ") + } + + sb.WriteString("(") + for i2, dst := range out.Dst { + if i2 > 0 { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("%v", nodeIDs[dst])) + } + sb.WriteString(")") + } + sb.WriteString("\n") + } + + if node.InputValues().Len() > 0 { + sb.WriteString("VIn: ") + for i, in := range node.InputValues().Slots { + if i > 0 { + sb.WriteString(", ") + } + + if in == nil { + sb.WriteString("?") + } else { + sb.WriteString(fmt.Sprintf("%v", nodeIDs[in.Src])) + } + } + sb.WriteString("\n") + } + if node.OutputValues().Len() > 0 { + sb.WriteString("VOut: ") + for i, out := range node.OutputValues().Slots { + if i > 0 { + sb.WriteString(", ") + } + + sb.WriteString("(") + for i2, dst := range out.Dst { + if i2 > 0 { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("%v", nodeIDs[dst])) + } + sb.WriteString(")") + } + sb.WriteString("\n") + } + + } + return sb.String() +} + +func nodeTypeName(node Node) string { + typ := reflect2.TypeOfValue(node) + for typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + return typ.Name() +} + func AddNode[N Node](graph *Graph, typ N) N { graph.AddNode(typ) return typ From 2c1031f2a8065c7381b6c1bb35103ed74866e52d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 24 Apr 2025 16:47:17 +0800 Subject: [PATCH 126/155] =?UTF-8?q?=E8=A7=A3=E9=99=A4=E8=B4=A8=E6=8A=BC?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BB=93=E6=9D=9F=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go index 5817b54..c905db1 100644 --- a/sdks/blockchain/blockchain.go +++ b/sdks/blockchain/blockchain.go @@ -6,6 +6,7 @@ import ( "gitlink.org.cn/cloudream/common/utils/serder" "net/url" "strings" + "time" ) type InvokeReq struct { @@ -51,8 +52,9 @@ func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { } type UnPledgePointsReq struct { - PrivateKey string `form:"privateKey"` - BusinessCode string `form:"businessCode"` + PrivateKey string `form:"privateKey"` + BusinessCode string `form:"businessCode"` + EndTime time.Time `form:"endTime"` } func (c *Client) UnPledgePoints(req UnPledgePointsReq, token string) error { From a28bd0bd975ce07dc38581135dbfbd00eaad8531 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 25 Apr 2025 15:46:17 +0800 Subject: [PATCH 127/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=BB=9F=E8=AE=A1=E4=BF=A1=E6=81=AF=E7=9A=84?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/user.go | 22 ++++++++++++++++++++++ sdks/storage/models.go | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/sdks/storage/cdsapi/user.go b/sdks/storage/cdsapi/user.go index 2230451..01c8870 100644 --- a/sdks/storage/cdsapi/user.go +++ b/sdks/storage/cdsapi/user.go @@ -48,3 +48,25 @@ func (r *UserDeleteResp) ParseResponse(resp *http.Response) error { func (c *Client) UserDelete(req *UserDelete) error { return JSONAPINoData(c.cfg, http.DefaultClient, req) } + +const UserBatchGetStatsPath = "/v1/user/batchGetStats" + +type UserBatchGetStats struct { + UserIDs []cdssdk.UserID `json:"userIDs" binding:"required"` +} + +func (r *UserBatchGetStats) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, UserBatchGetStatsPath, r) +} + +type UserBatchGetStatsResp struct { + Stats []*cdssdk.UserStats `json:"stats"` +} + +func (r *UserBatchGetStatsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *Client) UserBatchGetStats(req *UserBatchGetStats) (*UserBatchGetStatsResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, req, &UserBatchGetStatsResp{}) +} diff --git a/sdks/storage/models.go b/sdks/storage/models.go index 9a53072..e1c622f 100644 --- a/sdks/storage/models.go +++ b/sdks/storage/models.go @@ -390,3 +390,9 @@ type PackageHash struct { // 16进制字符串格式的sha256哈希值 Hash string `json:"hash"` } + +type UserStats struct { + UserID UserID `json:"userID" gorm:"column:UserID"` + FileCount int64 `json:"fileCount" gorm:"column:FileCount"` + TotalSize int64 `json:"totalSize" gorm:"column:TotalSize"` +} From c4074cd83e4c4d6e6cc10a62c608f63012d92244 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Sun, 27 Apr 2025 09:47:43 +0800 Subject: [PATCH 128/155] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=9B=E5=BB=BApac?= =?UTF-8?q?kage=E8=B0=83=E5=BA=A6=E8=B7=AF=E5=BE=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index cd182d7..fc187e7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -156,14 +156,12 @@ func (d *DataBindingBase) Noop() {} type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - //Description string `json:"description"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - //PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` RepositoryName string `json:"repositoryName"` } From dd36c63595a7979b23be56b9061b65b7ba52443d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Sun, 27 Apr 2025 15:06:36 +0800 Subject: [PATCH 129/155] =?UTF-8?q?=E7=BB=91=E5=AE=9A=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=AF=B7=E6=B1=82=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index fc187e7..d545b78 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -156,13 +156,15 @@ func (d *DataBindingBase) Noop() {} type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - RepositoryName string `json:"repositoryName"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + ClusterIDs []string `json:"clusterIDs"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + RepositoryName string `json:"repositoryName"` } type HPCJobInfo struct { From e50dc3a1796f2892a883fa993a213c1390dffe41 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 28 Apr 2025 16:48:57 +0800 Subject: [PATCH 130/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=99=BA=E7=AE=97?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=8F=90=E4=BA=A4=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index d545b78..36804c6 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -125,11 +125,23 @@ type PCMInferenceJobInfo struct { type AIJobInfo struct { serder.Metadata `union:"AI"` JobInfoBase - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - Files JobFilesInfo `json:"files"` - JobResources JobResources `json:"jobResources"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` + ResourceChoice ResourceChoice `json:"resourceChoice"` +} + +type ResourceChoice struct { + Type string `json:"type"` + ResourceScopes []ResourceScope `json:"resourceScopes"` +} + +type ResourceScope struct { + Name string `json:"name"` + Min float64 `json:"min"` + Max float64 `json:"max"` } type BindingJobInfo struct { @@ -443,8 +455,9 @@ func (i *JobFileInfoBase) Noop() {} type BindingJobFileInfo struct { serder.Metadata `union:"Binding"` JobFileInfoBase - Type string `json:"type"` - BindingID int64 `json:"bindingID"` + Type string `json:"type"` + BindingID int64 `json:"bindingID"` + BindingName string `json:"bindingName"` } type PackageJobFileInfo struct { @@ -471,8 +484,9 @@ type DataReturnJobFileInfo struct { type ImageJobFileInfo struct { serder.Metadata `union:"Image"` JobFileInfoBase - Type string `json:"type"` - ImageID ImageID `json:"imageID"` + Type string `json:"type"` + ImageID ImageID `json:"imageID"` + ImageName string `json:"imageName"` } type JobRuntimeInfo struct { From 2edc2ae5e4d16809700522691f19c3ffe3d46113 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 30 Apr 2025 10:27:56 +0800 Subject: [PATCH 131/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E6=8F=90=E4=BA=A4=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 1 + sdks/scheduler/models.go | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 87a6cfe..1057c0d 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -115,6 +115,7 @@ type ModelDistribute struct { type CreateJobResp struct { TaskID TaskID `json:"taskID"` + TaskName string `json:"taskName"` ScheduleDatas []ScheduleData `json:"scheduleDatas"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 36804c6..f19f23a 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -455,8 +455,9 @@ func (i *JobFileInfoBase) Noop() {} type BindingJobFileInfo struct { serder.Metadata `union:"Binding"` JobFileInfoBase - Type string `json:"type"` - BindingID int64 `json:"bindingID"` + Type string `json:"type"` + BindingID int64 `json:"bindingID"` + // 用于参数回显 BindingName string `json:"bindingName"` } @@ -484,9 +485,10 @@ type DataReturnJobFileInfo struct { type ImageJobFileInfo struct { serder.Metadata `union:"Image"` JobFileInfoBase - Type string `json:"type"` - ImageID ImageID `json:"imageID"` - ImageName string `json:"imageName"` + Type string `json:"type"` + ImageID ImageID `json:"imageID"` + // 用于参数回显 + ImageName string `json:"imageName"` } type JobRuntimeInfo struct { From e137e20ff6418f3af95baa1aa9b4f71127f54eef Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 30 Apr 2025 15:52:21 +0800 Subject: [PATCH 132/155] =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E5=AE=8C?= =?UTF-8?q?=E5=96=84=E4=BE=9D=E8=B5=96=E5=85=B3=E7=B3=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index f19f23a..3f35cc3 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -51,6 +51,7 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string GetTargetLocalJobIDs() []string + SetTargetLocalJob(info TargetJobInfo) GetTargetInputParams(targetID string) map[string]string } @@ -70,9 +71,8 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` - //TargetLocalJobIDs []string `json:"targetLocalJobIDs"` - TargetJob []TargetJobInfo `json:"targetJob"` + LocalJobID string `json:"localJobID"` + TargetJob []TargetJobInfo `json:"targetJob"` } type TargetJobInfo struct { @@ -101,6 +101,16 @@ func (i *JobInfoBase) GetTargetLocalJobIDs() []string { return IDs } +func (i *JobInfoBase) SetTargetLocalJob(info TargetJobInfo) { + for _, target := range i.TargetJob { + // 已经存在，则不用再添加 + if target.TargetJobID == info.TargetJobID { + return + } + } + i.TargetJob = append(i.TargetJob, info) +} + type NormalJobInfo struct { serder.Metadata `union:"Normal"` JobInfoBase @@ -379,10 +389,9 @@ type DataPreprocessJobInfo struct { type DataReturnJobInfo struct { serder.Metadata `union:"DataReturn"` JobInfoBase - Type string `json:"type"` - BucketID cdssdk.BucketID `json:"bucketID"` - //BindingType string `json:"bindingType"` - TargetLocalJobID string `json:"targetLocalJobID"` + Type string `json:"type"` + BucketID cdssdk.BucketID `json:"bucketID"` + TargetLocalJobID string `json:"targetLocalJobID"` } // MultiInstanceJobInfo 多实例(推理任务) From af8678becfecb591f77870ccc9b383e98f254c20 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 30 Apr 2025 16:02:50 +0800 Subject: [PATCH 133/155] =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E5=AE=8C?= =?UTF-8?q?=E5=96=84=E4=BE=9D=E8=B5=96=E5=85=B3=E7=B3=BB2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 133b50e..2221f64 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -81,6 +81,9 @@ const ( Schedule = "schedule" BlockChainJobCreatePrefix = "job_create_" + + StartNode = "StartTask" + EndNode = "EndTask" ) type TaskID int64 From 3b89aef0a85d75c22525e131c2c8cb7fc6e5dbac Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 6 May 2025 09:09:49 +0800 Subject: [PATCH 134/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=BB=91=E5=AE=9A=E6=A8=A1=E5=9E=8B=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 6 +++--- sdks/pcmscheduler/models.go | 8 +++++--- sdks/scheduler/models.go | 18 +++++++++--------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 1057c0d..fa09286 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -61,7 +61,7 @@ type CreateInferenceJobResp struct { TaskId string `json:"taskId"` } -type CreatePCMJobReq struct { +type CreateAIJobReq struct { Name string `json:"name"` Description string `json:"description"` JobResources schsdk.JobResources `json:"jobResources"` @@ -126,7 +126,7 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } -func (c *Client) CreateInferenceJob(req CreatePCMJobReq, token string) (*CreateInferenceJobResp, error) { +func (c *Client) CreateInferenceJob(req CreateAIJobReq, token string) (*CreateInferenceJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") if err != nil { return nil, err @@ -160,7 +160,7 @@ func (c *Client) CreateInferenceJob(req CreatePCMJobReq, token string) (*CreateI } -func (c *Client) CreateJob(req CreatePCMJobReq, token string) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreateAIJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "schedule/createTask") if err != nil { return nil, err diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 2221f64..e18f0b6 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -282,8 +282,8 @@ func (d *JobSubmitInfoBase) Noop() {} type PCMJobSubmitInfo struct { serder.Metadata `union:"pcm"` JobSubmitInfoBase - Type string `json:"type"` - Info CreatePCMJobReq `json:"info"` + Type string `json:"type"` + Info CreateAIJobReq `json:"info"` } type HPCJobSubmitInfo struct { @@ -441,7 +441,9 @@ var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") -type DataBindingBase struct{} +type DataBindingBase struct { + RootPath string `json:"rootPath"` +} func (d *DataBindingBase) Noop() {} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 3f35cc3..8ab173b 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -178,15 +178,15 @@ func (d *DataBindingBase) Noop() {} type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - ClusterIDs []string `json:"clusterIDs"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - RepositoryName string `json:"repositoryName"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + ClusterIDs []ClusterID `json:"clusterIDs"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + RepositoryName string `json:"repositoryName"` } type HPCJobInfo struct { From 1df1ee6660df4399f4b7cadad2ddc8fd7d2b003b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 6 May 2025 10:11:37 +0800 Subject: [PATCH 135/155] =?UTF-8?q?=E5=8A=9F=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/future/future.go | 6 +++++- pkgs/future/ready.go | 2 +- pkgs/future/set_value_future.go | 18 ++++++++++++++++-- pkgs/future/set_void_future.go | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/pkgs/future/future.go b/pkgs/future/future.go index 767447c..0f952b4 100644 --- a/pkgs/future/future.go +++ b/pkgs/future/future.go @@ -5,7 +5,11 @@ import ( "fmt" ) -var ErrCompleted = fmt.Errorf("context canceled") +var ErrConsumed = fmt.Errorf("future already consumed") + +var ErrNotComplete = fmt.Errorf("future not complete") + +var ErrCanceled = context.Canceled type Future interface { IsComplete() bool diff --git a/pkgs/future/ready.go b/pkgs/future/ready.go index 056ec2c..70594af 100644 --- a/pkgs/future/ready.go +++ b/pkgs/future/ready.go @@ -24,7 +24,7 @@ func (f *Ready) Wait(ctx context.Context) error { select { case v, ok := <-f.ch: if !ok { - return ErrCompleted + return ErrConsumed } return v diff --git a/pkgs/future/set_value_future.go b/pkgs/future/set_value_future.go index d59aa17..43d0d14 100644 --- a/pkgs/future/set_value_future.go +++ b/pkgs/future/set_value_future.go @@ -63,7 +63,7 @@ func (f *SetValueFuture[T]) Wait(ctx context.Context) (T, error) { case cv, ok := <-f.ch: if !ok { var ret T - return ret, cv.Err + return ret, ErrConsumed } return cv.Value, cv.Err @@ -73,6 +73,20 @@ func (f *SetValueFuture[T]) Wait(ctx context.Context) (T, error) { } } +func (f *SetValueFuture[T]) TryGetValue() (T, error) { + select { + case cv, ok := <-f.ch: + if !ok { + var ret T + return ret, ErrConsumed + } + return cv.Value, cv.Err + default: + var ret T + return ret, ErrNotComplete + } +} + type SetValueFuture2[T1 any, T2 any] struct { isCompleted bool ch chan ChanValue2[T1, T2] @@ -126,7 +140,7 @@ func (f *SetValueFuture2[T1, T2]) Wait(ctx context.Context) (T1, T2, error) { select { case cv, ok := <-f.ch: if !ok { - return cv.Value1, cv.Value2, cv.Err + return cv.Value1, cv.Value2, ErrConsumed } return cv.Value1, cv.Value2, cv.Err diff --git a/pkgs/future/set_void_future.go b/pkgs/future/set_void_future.go index d4fd0b0..2febe92 100644 --- a/pkgs/future/set_void_future.go +++ b/pkgs/future/set_void_future.go @@ -41,7 +41,7 @@ func (f *SetVoidFuture) Wait(ctx context.Context) error { select { case v, ok := <-f.ch: if !ok { - return ErrCompleted + return ErrConsumed } return v From d7b1738ee986f6b829dece3db5da0439a58f76a3 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 6 May 2025 16:37:27 +0800 Subject: [PATCH 136/155] =?UTF-8?q?=E5=9B=9E=E6=BA=90=E5=90=8E=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E4=B8=BA=E6=A8=A1=E5=9E=8B=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index e18f0b6..cd0b189 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -31,6 +31,7 @@ const ( PackageTypeNormal = "normal" PackageTypeNull = "null" + Null = "null" OrderByName = "name" OrderBySize = "size" From c8e1fcab4558c49af6ddb3b1154511f30934daf1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 7 May 2025 10:49:07 +0800 Subject: [PATCH 137/155] =?UTF-8?q?=E8=A7=84=E8=8C=83mysql=E8=A1=A8?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/models.go | 4 ++-- sdks/pcmscheduler/models.go | 4 ++-- sdks/uploader/models.go | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sdks/hpc/models.go b/sdks/hpc/models.go index 6511bb9..cee9586 100644 --- a/sdks/hpc/models.go +++ b/sdks/hpc/models.go @@ -466,7 +466,7 @@ type ClusterImage struct { } func (ClusterImage) TableName() string { - return "clusterImage" + return "cluster_image" } type ClusterImageCard struct { @@ -475,7 +475,7 @@ type ClusterImageCard struct { } func (ClusterImageCard) TableName() string { - return "clusterImageCard" + return "cluster_image_card" } type QueryBindingFilters struct { diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index cd0b189..e2c6bc4 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -536,7 +536,7 @@ type ClusterImage struct { } func (ClusterImage) TableName() string { - return "clusterImage" + return "cluster_image" } type ClusterImageCard struct { @@ -545,7 +545,7 @@ type ClusterImageCard struct { } func (ClusterImageCard) TableName() string { - return "clusterImageCard" + return "cluster_image_card" } type QueryBindingFilters struct { diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 35ebf1d..9ad3f8d 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -21,7 +21,7 @@ type BlockChain struct { } func (BlockChain) TableName() string { - return "BlockChain" // 确保和数据库中的表名一致 + return "block_chain" // 确保和数据库中的表名一致 } type Binding struct { @@ -73,7 +73,7 @@ type BindingDetail struct { } func (Binding) TableName() string { - return "BindingData" // 确保和数据库中的表名一致 + return "bindings" // 确保和数据库中的表名一致 } type BindingCluster struct { @@ -85,7 +85,7 @@ type BindingCluster struct { } func (BindingCluster) TableName() string { - return "bindingCluster" // 确保和数据库中的表名一致 + return "binding_cluster" // 确保和数据库中的表名一致 } type Folder struct { @@ -108,7 +108,7 @@ type Cluster struct { } func (Cluster) TableName() string { - return "uploadedCluster" // 确保和数据库中的表名一致 + return "uploaded_cluster" // 确保和数据库中的表名一致 } type Package struct { @@ -157,7 +157,7 @@ type PackageCloneDAO struct { } func (PackageCloneDAO) TableName() string { - return "packageClone" // 确保和数据库中的表名一致 + return "package_clone" // 确保和数据库中的表名一致 } type PackageCloneParam struct { @@ -195,7 +195,7 @@ type ClusterMapping struct { } func (ClusterMapping) TableName() string { - return "ClusterMapping" + return "cluster_mapping" } type ScheduleTarget interface { From fd3ff190fd793bb7d8e30149cca04ea3843793e9 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 7 May 2025 11:36:53 +0800 Subject: [PATCH 138/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Const=E8=8A=82?= =?UTF-8?q?=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/ioswitch/plan/ops/var.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkgs/ioswitch/plan/ops/var.go b/pkgs/ioswitch/plan/ops/var.go index 962ec4c..973cc59 100644 --- a/pkgs/ioswitch/plan/ops/var.go +++ b/pkgs/ioswitch/plan/ops/var.go @@ -1,6 +1,7 @@ package ops import ( + "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" ) @@ -21,3 +22,31 @@ func (o *ConstVar) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ConstVar) String() string { return "ConstVar" } + +type ConstNode struct { + dag.NodeBase + Value exec.VarValue +} + +func (b *GraphNodeBuilder) NewConst(val exec.VarValue) *ConstNode { + node := &ConstNode{ + Value: val, + } + b.AddNode(node) + + node.OutputValues().Init(node, 1) + return node +} + +func (t *ConstNode) Output() dag.ValueOutputSlot { + return dag.ValueOutputSlot{ + Node: t, + Index: 0, + } +} + +func (t *ConstNode) GenerateOp() (exec.Op, error) { + return &DropStream{ + Input: t.Output().Var().VarID, + }, nil +} From a059fc72947cbf986e8e3647eaa5b8ed53e8a1d1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 7 May 2025 17:40:23 +0800 Subject: [PATCH 139/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=8A=B6=E6=80=81=E6=9F=A5=E8=AF=A2=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 9 +++++++++ sdks/scheduler/jobflow.go | 28 ++++++++++++++++++++++++++++ sdks/scheduler/models.go | 2 ++ 3 files changed, 39 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index e2c6bc4..5b10f04 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -56,6 +56,7 @@ const ( FailedStatus = "failed" SuccessStatus = "success" + SucceededStatus = "succeeded" UploadingStatus = "uploading" RunningStatus = "running" @@ -85,6 +86,14 @@ const ( StartNode = "StartTask" EndNode = "EndTask" + + NodeTypeBinding = "binding" + NodeTypeUpload = "upload" + NodeTypeDataReturn = "data_return" + NodeTypeHPCCreate = "hpc_create" + NodeTypeInference = "inference" + NodeTypeAICreate = "ai_job_create" + NodeTypeAIJobRun = "ai_job_run" ) type TaskID int64 diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index 5219b80..485e9b0 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -67,3 +67,31 @@ type JobFlow struct { UpdatedAt time.Time `json:"updatedAt"` CreatedAt time.Time `json:"createdAt"` } + +type JobFlowRunStatus struct { + RunID JobSetID `gorm:"column:run_id;primaryKey" json:"runID"` + NodeType string `gorm:"column:node_type;size:100" json:"nodeType"` + NodeID string `gorm:"column:node_id;size:255" json:"nodeID"` + Status string `gorm:"column:status;type:enum('pending','running','fail','success')" json:"status"` + RunLog string `gorm:"column:run_log;type:text" json:"runLog"` +} + +type JobFlowRunDAO struct { + ID JobSetID `gorm:"column:id;primaryKey;" json:"runID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content string `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` +} + +type JobFlowRun struct { + ID JobSetID `gorm:"column:id;primaryKey;" json:"runID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content FlowData `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` +} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 8ab173b..7e36020 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -31,6 +31,8 @@ type JobID string type JobSetID string +type DataID int64 + type ImageID int64 // 计算中心ID From 7cf21f30837f554da4b31c4e35dc528e3816529e Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 8 May 2025 16:48:37 +0800 Subject: [PATCH 140/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 7e36020..157f6bc 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -13,6 +13,7 @@ const ( JobTypeInstance = "Instance" JobTypeFinetuning = "Finetuning" JobTypeDataPreprocess = "DataPreprocess" + JobTypeDataReturn = "DataReturn" FileInfoTypePackage = "Package" FileInfoTypeLocalFile = "LocalFile" @@ -132,6 +133,8 @@ type PCMInferenceJobInfo struct { Description string `json:"description"` Files JobFilesInfo `json:"files"` JobResources JobResources `json:"jobResources"` + + BindingName string `json:"bindingName"` } type AIJobInfo struct { @@ -161,6 +164,7 @@ type BindingJobInfo struct { JobInfoBase Type string `json:"type"` Info DataBinding `json:"info"` + Name string `json:"name"` // 临时使用 } type DataBinding interface { @@ -394,6 +398,16 @@ type DataReturnJobInfo struct { Type string `json:"type"` BucketID cdssdk.BucketID `json:"bucketID"` TargetLocalJobID string `json:"targetLocalJobID"` + ReportMessage []ReportMessage `json:"reportMessage"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID ClusterID `json:"clusterID"` + Output string `json:"output"` } // MultiInstanceJobInfo 多实例(推理任务) From c8b2ef2afca71759024f0216ba19fb80f77e2301 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 9 May 2025 09:49:40 +0800 Subject: [PATCH 141/155] =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E5=8F=98=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 157f6bc..01a4fcb 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -70,6 +70,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*HPCJobInfo)(nil), (*BindingJobInfo)(nil), (*PCMInferenceJobInfo)(nil), + (*FinishJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -148,6 +149,12 @@ type AIJobInfo struct { ResourceChoice ResourceChoice `json:"resourceChoice"` } +type FinishJobInfo struct { + serder.Metadata `union:"finish"` + JobInfoBase + Type string `json:"type"` +} + type ResourceChoice struct { Type string `json:"type"` ResourceScopes []ResourceScope `json:"resourceScopes"` From 4e1a3cb8807af297316bdc7a36ee3fa1f7e81e16 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 9 May 2025 15:19:39 +0800 Subject: [PATCH 142/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E8=BE=93=E5=87=BA=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/jobflow.go | 13 ++++---- sdks/scheduler/models.go | 62 ++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index 485e9b0..1437598 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -69,11 +69,12 @@ type JobFlow struct { } type JobFlowRunStatus struct { - RunID JobSetID `gorm:"column:run_id;primaryKey" json:"runID"` - NodeType string `gorm:"column:node_type;size:100" json:"nodeType"` - NodeID string `gorm:"column:node_id;size:255" json:"nodeID"` - Status string `gorm:"column:status;type:enum('pending','running','fail','success')" json:"status"` - RunLog string `gorm:"column:run_log;type:text" json:"runLog"` + RunID JobSetID `gorm:"column:run_id;primaryKey" json:"runID"` + NodeType string `gorm:"column:node_type;size:100" json:"nodeType"` + NodeID string `gorm:"column:node_id;size:255" json:"nodeID"` + Status string `gorm:"column:status;type:enum('pending','running','fail','success')" json:"status"` + RunOutput string `gorm:"column:run_output;type:text" json:"runOutput"` + RunLog string `gorm:"column:run_log;type:text" json:"runLog"` } type JobFlowRunDAO struct { @@ -84,6 +85,7 @@ type JobFlowRunDAO struct { Content string `gorm:"column:content;type:text" json:"content"` Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` + FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` } type JobFlowRun struct { @@ -94,4 +96,5 @@ type JobFlowRun struct { Content FlowData `gorm:"column:content;type:text" json:"content"` Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` + FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 01a4fcb..f3a920b 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -134,8 +134,7 @@ type PCMInferenceJobInfo struct { Description string `json:"description"` Files JobFilesInfo `json:"files"` JobResources JobResources `json:"jobResources"` - - BindingName string `json:"bindingName"` + BindingID DataID `json:"bindingID"` } type AIJobInfo struct { @@ -408,15 +407,6 @@ type DataReturnJobInfo struct { ReportMessage []ReportMessage `json:"reportMessage"` } -type ReportMessage struct { - TaskName string `json:"taskName"` - TaskID string `json:"taskID"` - Status bool `json:"status"` - Message string `json:"message"` - ClusterID ClusterID `json:"clusterID"` - Output string `json:"output"` -} - // MultiInstanceJobInfo 多实例(推理任务) type MultiInstanceJobInfo struct { serder.Metadata `union:"MultiInstance"` @@ -629,3 +619,53 @@ type InferencePlatform struct { SimilarityThreshold string `json:"similarityThreshold"` EntriesPerFile string `json:"entriesPerFile"` } + +type JobOutput interface { + Output() +} + +var JobOutputTypeUnion = types.NewTypeUnion[JobOutput]( + (*AIJobOutput)(nil), + (*BindingJobOutput)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobOutputTypeUnion, "type") + +type JobOutputBase struct{} + +func (d *JobOutputBase) Output() {} + +type PublicOutput struct { + serder.Metadata `union:"object"` + JobOutputBase + Type string `json:"type"` +} + +type AIJobOutput struct { + serder.Metadata `union:"object"` + JobOutputBase + Type string `json:"type"` +} + +type BindingJobOutput struct { + serder.Metadata `union:"binding"` + JobOutputBase + Type string `json:"type"` + BindingID DataID `json:"bindingID"` +} + +type DataReturnJobOutput struct { + serder.Metadata `union:"DataReturn"` + JobOutputBase + Type string `json:"type"` + ReportMessage []ReportMessage `json:"reportMessage"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID ClusterID `json:"clusterID"` + Output string `json:"output"` +} From 5a93feb5a028ade3cf0ed03e8bf5d1ae54d00657 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 9 May 2025 15:31:32 +0800 Subject: [PATCH 143/155] fix2 --- sdks/scheduler/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index f3a920b..a5faad7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -149,7 +149,7 @@ type AIJobInfo struct { } type FinishJobInfo struct { - serder.Metadata `union:"finish"` + serder.Metadata `union:"Finish"` JobInfoBase Type string `json:"type"` } From fda3a4ca9aeedcccddcffb947dbe9d8d66348cf9 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 12 May 2025 10:00:56 +0800 Subject: [PATCH 144/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 3 +-- sdks/scheduler/models.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 5b10f04..ef01c8e 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -84,8 +84,7 @@ const ( BlockChainJobCreatePrefix = "job_create_" - StartNode = "StartTask" - EndNode = "EndTask" + Finish = "Finish" NodeTypeBinding = "binding" NodeTypeUpload = "upload" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index a5faad7..c40c16c 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -70,7 +70,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*HPCJobInfo)(nil), (*BindingJobInfo)(nil), (*PCMInferenceJobInfo)(nil), - (*FinishJobInfo)(nil), + (*CompleteJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -148,7 +148,7 @@ type AIJobInfo struct { ResourceChoice ResourceChoice `json:"resourceChoice"` } -type FinishJobInfo struct { +type CompleteJobInfo struct { serder.Metadata `union:"Finish"` JobInfoBase Type string `json:"type"` From 4be8100d1ab8cb42276ccc48b3a13ad8e2cea8ad Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 12 May 2025 15:32:22 +0800 Subject: [PATCH 145/155] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E5=88=86=E9=A1=B5=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 4 ++-- sdks/scheduler/models.go | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index ef01c8e..fc91a22 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -431,8 +431,8 @@ type QueryData struct { UserID cdssdk.UserID `json:"userID" binding:"required"` Path string `json:"path"` PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - CurrentPage int `json:"currentPage" binding:"required"` - PageSize int `json:"pageSize" binding:"required"` + CurrentPage int `json:"currentPage" binding:"gte=1"` // >= 1 + PageSize int `json:"pageSize" binding:"gte=1,lte=200"` // 1 ~ 200 OrderBy string `json:"orderBy" binding:"required"` PackageName string `json:"packageName"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index c40c16c..2f0ba77 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -129,12 +129,13 @@ type NormalJobInfo struct { type PCMInferenceJobInfo struct { serder.Metadata `union:"PCM_Inference"` JobInfoBase - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - Files JobFilesInfo `json:"files"` - JobResources JobResources `json:"jobResources"` - BindingID DataID `json:"bindingID"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` + BindingID DataID `json:"bindingID"` + ResourceChoice ResourceChoice `json:"resourceChoice"` } type AIJobInfo struct { From 7d232ef6258da9af08899021f8e2824cff198679 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 12 May 2025 16:18:42 +0800 Subject: [PATCH 146/155] fix --- sdks/pcmscheduler/models.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index fc91a22..dae0fa3 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -431,8 +431,8 @@ type QueryData struct { UserID cdssdk.UserID `json:"userID" binding:"required"` Path string `json:"path"` PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - CurrentPage int `json:"currentPage" binding:"gte=1"` // >= 1 - PageSize int `json:"pageSize" binding:"gte=1,lte=200"` // 1 ~ 200 + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` OrderBy string `json:"orderBy" binding:"required"` PackageName string `json:"packageName"` } From d01bb92ec90bb4ecbdfc5c732eb77017a6cdd1d9 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 13 May 2025 15:57:57 +0800 Subject: [PATCH 147/155] =?UTF-8?q?=E6=8E=A8=E7=90=86=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE=E8=B0=83=E5=BA=A6=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 2 +- sdks/scheduler/models.go | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index dae0fa3..6f8707f 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -84,7 +84,7 @@ const ( BlockChainJobCreatePrefix = "job_create_" - Finish = "Finish" + Complete = "Complete" NodeTypeBinding = "binding" NodeTypeUpload = "upload" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 2f0ba77..4038cc0 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -71,6 +71,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*BindingJobInfo)(nil), (*PCMInferenceJobInfo)(nil), (*CompleteJobInfo)(nil), + (*StartJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -150,7 +151,13 @@ type AIJobInfo struct { } type CompleteJobInfo struct { - serder.Metadata `union:"Finish"` + serder.Metadata `union:"Complete"` + JobInfoBase + Type string `json:"type"` +} + +type StartJobInfo struct { + serder.Metadata `union:"Start"` JobInfoBase Type string `json:"type"` } From befe803070b17eef3ba9ed50813574f99132e9a1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 14 May 2025 09:40:53 +0800 Subject: [PATCH 148/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=80=9A=E7=9F=A5?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 52 +++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 4038cc0..9e77968 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -409,10 +409,11 @@ type DataPreprocessJobInfo struct { type DataReturnJobInfo struct { serder.Metadata `union:"DataReturn"` JobInfoBase - Type string `json:"type"` - BucketID cdssdk.BucketID `json:"bucketID"` - TargetLocalJobID string `json:"targetLocalJobID"` - ReportMessage []ReportMessage `json:"reportMessage"` + Type string `json:"type"` + BucketID cdssdk.BucketID `json:"bucketID"` + TargetLocalJobID string `json:"targetLocalJobID"` + Report TrainJobStatusReport `json:"report"` + //ReportMessage []ReportMessage `json:"reportMessage"` } // MultiInstanceJobInfo 多实例(推理任务) @@ -665,8 +666,9 @@ type BindingJobOutput struct { type DataReturnJobOutput struct { serder.Metadata `union:"DataReturn"` JobOutputBase - Type string `json:"type"` - ReportMessage []ReportMessage `json:"reportMessage"` + Type string `json:"type"` + Report TrainJobStatusReport `json:"report"` + //ReportMessage []ReportMessage `json:"reportMessage"` } type ReportMessage struct { @@ -677,3 +679,41 @@ type ReportMessage struct { ClusterID ClusterID `json:"clusterID"` Output string `json:"output"` } + +type JobStatusReport interface { + Report() +} + +var JobStatusReportTypeUnion = types.NewTypeUnion[JobStatusReport]( + (*TrainJobStatusReport)(nil), + (*InferenceJobStatusReport)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobStatusReportTypeUnion, "type") + +type JobStatusReportBase struct{} + +func (d *JobStatusReportBase) Report() {} + +type TrainJobStatusReport struct { + serder.Metadata `union:"Train"` + JobStatusReportBase + Type string `json:"type"` + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID ClusterID `json:"clusterID"` + Output string `json:"output"` +} + +type InferenceJobStatusReport struct { + serder.Metadata `union:"Inference"` + JobStatusReportBase + Type string `json:"type"` + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + URL string `json:"url"` +} From b3ed5f8140cf95ba06327b9002cafd95093feed0 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 14 May 2025 15:39:02 +0800 Subject: [PATCH 149/155] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9e77968..60261cc 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -666,18 +666,8 @@ type BindingJobOutput struct { type DataReturnJobOutput struct { serder.Metadata `union:"DataReturn"` JobOutputBase - Type string `json:"type"` - Report TrainJobStatusReport `json:"report"` - //ReportMessage []ReportMessage `json:"reportMessage"` -} - -type ReportMessage struct { - TaskName string `json:"taskName"` - TaskID string `json:"taskID"` - Status bool `json:"status"` - Message string `json:"message"` - ClusterID ClusterID `json:"clusterID"` - Output string `json:"output"` + Type string `json:"type"` + ReportMessage TrainJobStatusReport `json:"report"` } type JobStatusReport interface { From 034e9d4768b8103dad3af02bd8527d159fb37597 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 14 May 2025 17:38:50 +0800 Subject: [PATCH 150/155] fix --- sdks/scheduler/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 60261cc..9dca84d 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -412,7 +412,7 @@ type DataReturnJobInfo struct { Type string `json:"type"` BucketID cdssdk.BucketID `json:"bucketID"` TargetLocalJobID string `json:"targetLocalJobID"` - Report TrainJobStatusReport `json:"report"` + ReportMessage TrainJobStatusReport `json:"report"` //ReportMessage []ReportMessage `json:"reportMessage"` } From b916bbac59373c8d2b4b81ed85b7dc8b5bc7e588 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 16 May 2025 17:45:28 +0800 Subject: [PATCH 151/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=8A=B6=E6=80=81=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/jobflow.go | 2 ++ sdks/scheduler/models.go | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index 1437598..8d78ddf 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -84,6 +84,7 @@ type JobFlowRunDAO struct { Description string `gorm:"column:description;size:255" json:"description"` Content string `gorm:"column:content;type:text" json:"content"` Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + Token string `gorm:"column:token;size:255" json:"token"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` } @@ -95,6 +96,7 @@ type JobFlowRun struct { Description string `gorm:"column:description;size:255" json:"description"` Content FlowData `gorm:"column:content;type:text" json:"content"` Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + Token string `gorm:"column:token;size:255" json:"token"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9dca84d..54f5261 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -630,7 +630,7 @@ type InferencePlatform struct { } type JobOutput interface { - Output() + Output2() } var JobOutputTypeUnion = types.NewTypeUnion[JobOutput]( @@ -642,7 +642,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&JobOutputTypeUnion, "type") type JobOutputBase struct{} -func (d *JobOutputBase) Output() {} +func (d *JobOutputBase) Output2() {} type PublicOutput struct { serder.Metadata `union:"object"` @@ -650,10 +650,18 @@ type PublicOutput struct { Type string `json:"type"` } +type HPCOutput struct { + serder.Metadata `union:"HPCSlurm"` + JobOutputBase + Type string `json:"type"` + Output string `json:"output"` +} + type AIJobOutput struct { - serder.Metadata `union:"object"` + serder.Metadata `union:"AI"` JobOutputBase - Type string `json:"type"` + Type string `json:"type"` + Output string `json:"output"` } type BindingJobOutput struct { @@ -668,6 +676,7 @@ type DataReturnJobOutput struct { JobOutputBase Type string `json:"type"` ReportMessage TrainJobStatusReport `json:"report"` + PackageID cdssdk.PackageID `json:"packageID"` } type JobStatusReport interface { From 88484e929d07b15bd4969528d789332df570bb65 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 19 May 2025 16:27:59 +0800 Subject: [PATCH 152/155] =?UTF-8?q?=E8=B0=83=E7=94=A8C=E7=AB=AF=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E4=BC=A0=E5=85=A5token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 13 +++++++++++-- sdks/scheduler/models.go | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index fa09286..608cfe8 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -19,17 +19,26 @@ type GetClusterInfoReq struct { // TraceId string `json:"traceId"` //} -func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) { +func (c *Client) GetClusterInfo(req GetClusterInfoReq, token string) ([]ClusterDetail, error) { targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") if err != nil { return nil, err } //resp, err := http2.PostJSON(targetUrl, http2.RequestParam{Body: req}) - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{}) + header := map[string]string{ + "Authorization": token, + } + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Header: header, + }) if err != nil { return nil, err } + + //all, err := io.ReadAll(resp.Body) + //println(string(all)) + contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 54f5261..911bf2d 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -26,6 +26,10 @@ const ( MemoryUtilization = "MemoryUtilization" GPUUtilization = "GPUUtilization" CPUUtilization = "CPUUtilization" + + MethodPost = "POST" + MethodGet = "GET" + CodeSuccess = 200 ) type JobID string @@ -72,6 +76,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*PCMInferenceJobInfo)(nil), (*CompleteJobInfo)(nil), (*StartJobInfo)(nil), + (*NotifyJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -162,6 +167,16 @@ type StartJobInfo struct { Type string `json:"type"` } +type NotifyJobInfo struct { + serder.Metadata `union:"Notify"` + JobInfoBase + Type string `json:"type"` + RequestType string `json:"requestType"` + Url string `json:"url"` + Body any `json:"body"` + Headers map[string]string `json:"headers"` +} + type ResourceChoice struct { Type string `json:"type"` ResourceScopes []ResourceScope `json:"resourceScopes"` From fe10185b9855bae9079ab72d90612d2bfdd75f16 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 20 May 2025 10:08:51 +0800 Subject: [PATCH 153/155] fix --- sdks/pcmscheduler/jobset.go | 59 ++++++++++++++++++++++++------------- sdks/scheduler/models.go | 9 +++++- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 608cfe8..bcf329c 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -14,31 +14,21 @@ type GetClusterInfoReq struct { IDs []schsdk.ClusterID `json:"clusterIDs"` } -//type GetClusterInfoResp struct { -// Data []ClusterDetail `json:"data"` -// TraceId string `json:"traceId"` -//} - func (c *Client) GetClusterInfo(req GetClusterInfoReq, token string) ([]ClusterDetail, error) { targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") if err != nil { return nil, err } - //resp, err := http2.PostJSON(targetUrl, http2.RequestParam{Body: req}) - header := map[string]string{ - "Authorization": token, - } resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ - Header: header, + Header: map[string]string{ + "Authorization": token, + }, }) if err != nil { return nil, err } - //all, err := io.ReadAll(resp.Body) - //println(string(all)) - contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { @@ -166,7 +156,43 @@ func (c *Client) CreateInferenceJob(req CreateAIJobReq, token string) (*CreateIn } return nil, fmt.Errorf("unknow response content type: %s", contType) +} +type StopInferenceJobReq struct { + TaskId string `json:"taskId"` +} + +func (c *Client) StopInferenceJob(req StopInferenceJobReq, token string) error { + targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") + if err != nil { + return err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateInferenceJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return nil + } + + return fmt.Errorf("error: %s", codeResp.Message) + } + + return fmt.Errorf("unknow response content type: %s", contType) } func (c *Client) CreateJob(req CreateAIJobReq, token string) (*CreateJobResp, error) { @@ -175,13 +201,6 @@ func (c *Client) CreateJob(req CreateAIJobReq, token string) (*CreateJobResp, er return nil, err } - // 将req转换成json，并打印 - //req2, err := serder.ObjectToJSONEx(req) - //if err != nil { - // return nil, fmt.Errorf("request to json: %w", err) - //} - //fmt.Println(string(req2)) - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, Header: map[string]string{ diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 911bf2d..2073cf4 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -77,6 +77,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*CompleteJobInfo)(nil), (*StartJobInfo)(nil), (*NotifyJobInfo)(nil), + (*StopInferenceJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -144,6 +145,13 @@ type PCMInferenceJobInfo struct { ResourceChoice ResourceChoice `json:"resourceChoice"` } +type StopInferenceJobInfo struct { + serder.Metadata `union:"StopInference"` + JobInfoBase + Type string `json:"type"` + Url string `json:"url"` +} + type AIJobInfo struct { serder.Metadata `union:"AI"` JobInfoBase @@ -428,7 +436,6 @@ type DataReturnJobInfo struct { BucketID cdssdk.BucketID `json:"bucketID"` TargetLocalJobID string `json:"targetLocalJobID"` ReportMessage TrainJobStatusReport `json:"report"` - //ReportMessage []ReportMessage `json:"reportMessage"` } // MultiInstanceJobInfo 多实例(推理任务) From 921a874200125691488831513b8c1ff710c844cb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 20 May 2025 17:11:54 +0800 Subject: [PATCH 154/155] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 31 ++++++++ sdks/uploader/uploader.go | 145 ++------------------------------------ utils/reflect2/assign.go | 29 ++++++++ 3 files changed, 67 insertions(+), 138 deletions(-) create mode 100644 utils/reflect2/assign.go diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 2073cf4..8262d75 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -78,6 +78,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*StartJobInfo)(nil), (*NotifyJobInfo)(nil), (*StopInferenceJobInfo)(nil), + (*UploadJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -175,6 +176,13 @@ type StartJobInfo struct { Type string `json:"type"` } +type UploadJobInfo struct { + serder.Metadata `union:"Upload"` + JobInfoBase + Type string `json:"type"` + DataType string `json:"dataType"` +} + type NotifyJobInfo struct { serder.Metadata `union:"Notify"` JobInfoBase @@ -210,6 +218,7 @@ type DataBinding interface { var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( (*ModelBinding)(nil), + (*DatasetBinding)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") @@ -232,6 +241,20 @@ type ModelBinding struct { RepositoryName string `json:"repositoryName"` } +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + OperateType string `json:"operateType"` + ClusterIDs []ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints int64 `json:"points"` +} + type HPCJobInfo struct { serder.Metadata `union:"HPC"` JobInfoBase @@ -658,6 +681,7 @@ type JobOutput interface { var JobOutputTypeUnion = types.NewTypeUnion[JobOutput]( (*AIJobOutput)(nil), (*BindingJobOutput)(nil), + (*UploadJobOutput)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobOutputTypeUnion, "type") @@ -693,6 +717,13 @@ type BindingJobOutput struct { BindingID DataID `json:"bindingID"` } +type UploadJobOutput struct { + serder.Metadata `union:"upload"` + JobOutputBase + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` +} + type DataReturnJobOutput struct { serder.Metadata `union:"DataReturn"` JobOutputBase diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go index 54f17df..6cae186 100644 --- a/sdks/uploader/uploader.go +++ b/sdks/uploader/uploader.go @@ -2,152 +2,21 @@ package uploadersdk import ( "fmt" - "gitlink.org.cn/cloudream/common/pkgs/types" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" + "mime/multipart" "net/url" "strings" ) -//type DataScheduleReq struct { -// PackageID cdssdk.PackageID `json:"packageID"` -// DataType string `json:"dataType"` -// ScheduleTarget ScheduleTarget `json:"scheduleTarget"` -//} - -//type DataScheduleResp struct { -// Results []sch.DataScheduleResult `json:"data"` -//} - -//type TmpDataScheduleResult struct { -// Cluster sch.DataDetail `json:"cluster"` -// PackageID cdssdk.PackageID `json:"packageID"` -// Status bool `json:"status"` -// Msg string `json:"msg"` -//} - -//func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) { -// targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule") -// if err != nil { -// return nil, err -// } -// -// resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ -// Body: req, -// }) -// if err != nil { -// return nil, err -// } -// println(resp.Body) -// -// contType := resp.Header.Get("Content-Type") -// if strings.Contains(contType, http2.ContentTypeJSON) { -// var codeResp response[[]TmpDataScheduleResult] -// if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { -// return nil, fmt.Errorf("parsing response: %w", err) -// } -// -// if codeResp.Code == ResponseCodeOK { -// var results []sch.DataScheduleResult -// for _, tmpResult := range codeResp.Data { -// result := sch.DataScheduleResult{ -// PackageID: tmpResult.PackageID, -// Status: tmpResult.Status, -// Msg: tmpResult.Msg, -// Clusters: []sch.DataDetail{ -// tmpResult.Cluster, -// }, -// } -// results = append(results, result) -// } -// return results, nil -// } -// -// return nil, codeResp.ToError() -// } -// -// return nil, fmt.Errorf("unknow response content type: %s", contType) -//} - -type UploadReq struct { - DataType string `json:"dataType"` - Source UploadSource `json:"source"` - Target UploadTarget `json:"target"` - //StorageIDs []cdssdk.StorageID `json:"storageIDs"` -} - -type UploadSource interface { - Noop() -} - -var UploadSourceTypeUnion = types.NewTypeUnion[UploadSource]( - (*PackageSource)(nil), - (*UrlSource)(nil), -) - -var _ = serder.UseTypeUnionInternallyTagged(&UploadSourceTypeUnion, "type") - -type PackageSource struct { - serder.Metadata `union:"jcs"` - UploadSourceBase - Type string `json:"type"` - PackageID cdssdk.PackageID `json:"packageID"` -} - -type UrlSource struct { - serder.Metadata `union:"url"` - UploadSourceBase - Type string `json:"type"` - Url string `json:"url"` -} - -type UploadSourceBase struct{} - -func (d *UploadSourceBase) Noop() {} - -type UploadTarget interface { - Noop() -} - -var UploadTargetTypeUnion = types.NewTypeUnion[UploadTarget]( - (*UrlTarget)(nil), - (*ApiTarget)(nil), -) - -var _ = serder.UseTypeUnionInternallyTagged(&UploadTargetTypeUnion, "type") - -type UrlTarget struct { - serder.Metadata `union:"url"` - UploadTargetBase - Type string `json:"type"` - ClusterID ClusterID `json:"clusterId"` - JCSUploadInfo cdsapi.ObjectUploadInfo `form:"JCSUploadInfo"` -} - -type ApiTarget struct { - serder.Metadata `union:"api"` - UploadTargetBase - Type string `json:"type"` - Clusters []ClusterID `json:"clusters"` -} - -type UploadTargetBase struct{} - -func (d *UploadTargetBase) Noop() {} - -type UploadResp struct { - PackageID cdssdk.PackageID `json:"packageID"` - ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` - ClusterID ClusterID `json:"clusterID"` - JsonData string `json:"jsonData"` - Status bool `json:"status"` - Message string `json:"message"` +type ObjectUploadReq struct { + Info cdsapi.ObjectUploadInfo `form:"info" binding:"required"` + Files []*multipart.FileHeader `form:"files"` } -func (c *Client) Upload(req UploadReq) (*UploadResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/dataUpload") +func (c *Client) Upload(req ObjectUploadReq) (*cdsapi.ObjectUploadResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/object/upload") if err != nil { return nil, err } @@ -161,7 +30,7 @@ func (c *Client) Upload(req UploadReq) (*UploadResp, error) { contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[UploadResp] + var codeResp response[cdsapi.ObjectUploadResp] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { return nil, fmt.Errorf("parsing response: %w", err) } diff --git a/utils/reflect2/assign.go b/utils/reflect2/assign.go new file mode 100644 index 0000000..2e8d49d --- /dev/null +++ b/utils/reflect2/assign.go @@ -0,0 +1,29 @@ +package reflect2 + +import "reflect" + +// MergeNonZero 将 src 中非零值字段复制到 dst 中 +func MergeNonZero(dst, src any) { + dstVal := reflect.ValueOf(dst).Elem() + srcVal := reflect.ValueOf(src).Elem() + + for i := 0; i < dstVal.NumField(); i++ { + dstField := dstVal.Field(i) + srcField := srcVal.Field(i) + + // 如果 src 字段是零值，则跳过 + if isZero(srcField) { + continue + } + + // 如果 dst 字段可设置，才设置 + if dstField.CanSet() { + dstField.Set(srcField) + } + } +} + +func isZero(v reflect.Value) bool { + zero := reflect.Zero(v.Type()) + return reflect.DeepEqual(v.Interface(), zero.Interface()) +} From a59532e284884f57c871cd02fe8a6fde8f5d2a19 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 27 May 2025 10:06:21 +0800 Subject: [PATCH 155/155] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/ioswitch/dag/node.go | 30 +++++++++++++++++++++++++++--- pkgs/ioswitch/dag/var.go | 5 +++++ pkgs/ioswitch/plan/compile.go | 4 ++-- pkgs/ioswitch/plan/ops/store.go | 2 +- pkgs/ioswitch/plan/ops/sync.go | 2 +- 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/pkgs/ioswitch/dag/node.go b/pkgs/ioswitch/dag/node.go index d513267..df6e801 100644 --- a/pkgs/ioswitch/dag/node.go +++ b/pkgs/ioswitch/dag/node.go @@ -20,24 +20,28 @@ type NodeEnv struct { Pinned bool // 如果为true，则不应该改变这个节点的执行环境 } -func (e *NodeEnv) ToEnvUnknown() { +func (e *NodeEnv) ToEnvUnknown(pinned bool) { e.Type = EnvUnknown e.Worker = nil + e.Pinned = pinned } -func (e *NodeEnv) ToEnvDriver() { +func (e *NodeEnv) ToEnvDriver(pinned bool) { e.Type = EnvDriver e.Worker = nil + e.Pinned = pinned } -func (e *NodeEnv) ToEnvWorker(worker exec.WorkerInfo) { +func (e *NodeEnv) ToEnvWorker(worker exec.WorkerInfo, pinned bool) { e.Type = EnvWorker e.Worker = worker + e.Pinned = pinned } func (e *NodeEnv) CopyFrom(other *NodeEnv) { e.Type = other.Type e.Worker = other.Worker + e.Pinned = other.Pinned } func (e *NodeEnv) Equals(other *NodeEnv) bool { @@ -461,6 +465,16 @@ func (s StreamOutputSlot) ToSlot(slot StreamInputSlot) { s.Var().To(slot.Node, slot.Index) } +// 查询所有输出的连接的输入槽位 +func (s StreamOutputSlot) ListDstSlots() []StreamInputSlot { + slots := make([]StreamInputSlot, s.Var().Dst.Len()) + myVar := s.Var() + for i, dst := range s.Var().Dst { + slots[i] = StreamInputSlot{Node: dst, Index: dst.InputStreams().IndexOf(myVar)} + } + return slots +} + type StreamInputSlot struct { Node Node Index int @@ -483,6 +497,16 @@ func (s ValueOutputSlot) ToSlot(slot ValueInputSlot) { s.Var().To(slot.Node, slot.Index) } +// 查询所有输出的连接的输入槽位 +func (s ValueOutputSlot) ListDstSlots() []ValueInputSlot { + slots := make([]ValueInputSlot, s.Var().Dst.Len()) + myVar := s.Var() + for i, dst := range s.Var().Dst { + slots[i] = ValueInputSlot{Node: dst, Index: dst.InputValues().IndexOf(myVar)} + } + return slots +} + type ValueInputSlot struct { Node Node Index int diff --git a/pkgs/ioswitch/dag/var.go b/pkgs/ioswitch/dag/var.go index 5d5989e..217def7 100644 --- a/pkgs/ioswitch/dag/var.go +++ b/pkgs/ioswitch/dag/var.go @@ -64,6 +64,11 @@ func (v *ValueVar) To(to Node, slotIdx int) { to.InputValues().Slots.Set(slotIdx, v) } +func (v *ValueVar) ToSlot(slot ValueInputSlot) { + v.Dst.Add(slot.Node) + slot.Node.InputValues().Slots.Set(slot.Index, v) +} + func (v *ValueVar) NotTo(node Node) { v.Dst.Remove(node) node.InputValues().Slots.Clear(v) diff --git a/pkgs/ioswitch/plan/compile.go b/pkgs/ioswitch/plan/compile.go index 6e75abe..bcdf20c 100644 --- a/pkgs/ioswitch/plan/compile.go +++ b/pkgs/ioswitch/plan/compile.go @@ -42,7 +42,7 @@ func generateSend(graph *ops.GraphNodeBuilder) { dstNode := out.Dst.Get(0) getNode := graph.NewGetStream(node.Env().Worker) - getNode.Env().ToEnvDriver() + getNode.Env().ToEnvDriver(true) // // 同时需要对此变量生成HoldUntil指令，避免Plan结束时Get指令还未到达 holdNode := graph.NewHoldUntil() @@ -86,7 +86,7 @@ func generateSend(graph *ops.GraphNodeBuilder) { // // 如果是要送到Driver，则只能由Driver主动去拉取 dstNode := out.Dst.Get(0) getNode := graph.NewGetValue(node.Env().Worker) - getNode.Env().ToEnvDriver() + getNode.Env().ToEnvDriver(true) // // 同时需要对此变量生成HoldUntil指令，避免Plan结束时Get指令还未到达 holdNode := graph.NewHoldUntil() diff --git a/pkgs/ioswitch/plan/ops/store.go b/pkgs/ioswitch/plan/ops/store.go index 4353dc7..caf45e2 100644 --- a/pkgs/ioswitch/plan/ops/store.go +++ b/pkgs/ioswitch/plan/ops/store.go @@ -23,7 +23,7 @@ func (o *Store) Execute(ctx *exec.ExecContext, e *exec.Executor) error { } func (o *Store) String() string { - return fmt.Sprintf("Store %v: %v", o.Key, o.Var) + return fmt.Sprintf("Store %v as \"%v\"", o.Var, o.Key) } type StoreConst struct { diff --git a/pkgs/ioswitch/plan/ops/sync.go b/pkgs/ioswitch/plan/ops/sync.go index c4ec53c..3e096cb 100644 --- a/pkgs/ioswitch/plan/ops/sync.go +++ b/pkgs/ioswitch/plan/ops/sync.go @@ -113,7 +113,7 @@ func (w *HoldUntil) Execute(ctx *exec.ExecContext, e *exec.Executor) error { } func (w *HoldUntil) String() string { - return fmt.Sprintf("HoldUntil Waits: %v, (%v) -> (%v)", utils.FormatVarIDs(w.Waits), utils.FormatVarIDs(w.Holds), utils.FormatVarIDs(w.Emits)) + return fmt.Sprintf("HoldUntil(waits=%v): %v -> %v", utils.FormatVarIDs(w.Waits), utils.FormatVarIDs(w.Holds), utils.FormatVarIDs(w.Emits)) } type HangUntil struct {