diff --git a/consts/errorcode/error_code.go b/consts/errorcode/error_code.go index 984ca9c..8dffeb3 100644 --- a/consts/errorcode/error_code.go +++ b/consts/errorcode/error_code.go @@ -7,4 +7,5 @@ const ( DataExists = "DataExists" BadArgument = "BadArgument" TaskNotFound = "TaskNotFound" + Unauthorized = "Unauthorized" ) diff --git a/go.mod b/go.mod index 16bebdb..00dc24a 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,14 @@ module gitlink.org.cn/cloudream/common -go 1.20 +go 1.23.0 + +toolchain go1.23.2 require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 - github.com/google/uuid v1.3.0 + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/credentials v1.17.62 + github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.15 github.com/json-iterator/go v1.1.12 @@ -12,38 +16,41 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/modern-go/reflect2 v1.0.2 github.com/otiai10/copy v1.12.0 - github.com/samber/lo v1.36.0 - github.com/sirupsen/logrus v1.9.2 - github.com/smartystreets/goconvey v1.8.0 + github.com/samber/lo v1.49.1 + github.com/sirupsen/logrus v1.9.3 + github.com/smartystreets/goconvey v1.8.1 github.com/streadway/amqp v1.1.0 github.com/zyedidia/generic v1.2.1 - go.etcd.io/etcd/client/v3 v3.5.9 - golang.org/x/exp v0.0.0-20230519143937-03e91628a987 + go.etcd.io/etcd/client/v3 v3.5.12 + golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa +) + +require ( + github.com/aws/smithy-go v1.22.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/smarty/assertions v1.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect ) require ( - github.com/benbjohnson/clock v1.3.0 // indirect - github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-querystring v1.1.0 github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/smartystreets/assertions v1.13.1 // indirect - github.com/stretchr/testify v1.8.2 // indirect - go.etcd.io/etcd/api/v3 v3.5.9 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/goleak v1.1.12 // indirect - go.uber.org/multierr v1.9.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect - google.golang.org/grpc v1.54.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + go.etcd.io/etcd/api/v3 v3.5.12 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect ) diff --git a/go.sum b/go.sum index 60a4de8..58cfdfc 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,32 @@ github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ= github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBajbBWGqjNTCa79ocU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -35,119 +42,96 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw= -github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= -github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= -github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= -github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= +github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM= github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= -go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= -go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= -go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= -go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= -go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= -go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= +go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= +go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= +go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= +go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA= -golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkgs/bitmap/bitmap.go b/pkgs/bitmap/bitmap.go index b1c3e63..d655772 100644 --- a/pkgs/bitmap/bitmap.go +++ b/pkgs/bitmap/bitmap.go @@ -23,7 +23,7 @@ func (b *Bitmap64) Weight() int { cnt := 0 for v > 0 { cnt++ - v &= (v - 1) + v &= v - 1 } return cnt } diff --git a/pkgs/future/future.go b/pkgs/future/future.go index 767447c..0f952b4 100644 --- a/pkgs/future/future.go +++ b/pkgs/future/future.go @@ -5,7 +5,11 @@ import ( "fmt" ) -var ErrCompleted = fmt.Errorf("context canceled") +var ErrConsumed = fmt.Errorf("future already consumed") + +var ErrNotComplete = fmt.Errorf("future not complete") + +var ErrCanceled = context.Canceled type Future interface { IsComplete() bool diff --git a/pkgs/future/ready.go b/pkgs/future/ready.go index 056ec2c..70594af 100644 --- a/pkgs/future/ready.go +++ b/pkgs/future/ready.go @@ -24,7 +24,7 @@ func (f *Ready) Wait(ctx context.Context) error { select { case v, ok := <-f.ch: if !ok { - return ErrCompleted + return ErrConsumed } return v diff --git a/pkgs/future/set_value_future.go b/pkgs/future/set_value_future.go index d59aa17..43d0d14 100644 --- a/pkgs/future/set_value_future.go +++ b/pkgs/future/set_value_future.go @@ -63,7 +63,7 @@ func (f *SetValueFuture[T]) Wait(ctx context.Context) (T, error) { case cv, ok := <-f.ch: if !ok { var ret T - return ret, cv.Err + return ret, ErrConsumed } return cv.Value, cv.Err @@ -73,6 +73,20 @@ func (f *SetValueFuture[T]) Wait(ctx context.Context) (T, error) { } } +func (f *SetValueFuture[T]) TryGetValue() (T, error) { + select { + case cv, ok := <-f.ch: + if !ok { + var ret T + return ret, ErrConsumed + } + return cv.Value, cv.Err + default: + var ret T + return ret, ErrNotComplete + } +} + type SetValueFuture2[T1 any, T2 any] struct { isCompleted bool ch chan ChanValue2[T1, T2] @@ -126,7 +140,7 @@ func (f *SetValueFuture2[T1, T2]) Wait(ctx context.Context) (T1, T2, error) { select { case cv, ok := <-f.ch: if !ok { - return cv.Value1, cv.Value2, cv.Err + return cv.Value1, cv.Value2, ErrConsumed } return cv.Value1, cv.Value2, cv.Err diff --git a/pkgs/future/set_void_future.go b/pkgs/future/set_void_future.go index d4fd0b0..2febe92 100644 --- a/pkgs/future/set_void_future.go +++ b/pkgs/future/set_void_future.go @@ -41,7 +41,7 @@ func (f *SetVoidFuture) Wait(ctx context.Context) error { select { case v, ok := <-f.ch: if !ok { - return ErrCompleted + return ErrConsumed } return v diff --git a/pkgs/ioswitch/dag/graph.go b/pkgs/ioswitch/dag/graph.go index 1a4e0f1..99ed675 100644 --- a/pkgs/ioswitch/dag/graph.go +++ b/pkgs/ioswitch/dag/graph.go @@ -1,7 +1,12 @@ package dag import ( + "fmt" + "reflect" + "strings" + "gitlink.org.cn/cloudream/common/utils/lo2" + "gitlink.org.cn/cloudream/common/utils/reflect2" ) type Graph struct { @@ -55,6 +60,100 @@ func (g *Graph) NewValueVar() *ValueVar { return &ValueVar{} } +func (g *Graph) Dump() string { + nodeIDs := make(map[Node]int) + for i, node := range g.Nodes { + nodeIDs[node] = i + } + + var sb strings.Builder + for _, node := range g.Nodes { + id, ok := nodeIDs[node] + if !ok { + id = len(nodeIDs) + nodeIDs[node] = id + } + sb.WriteString(fmt.Sprintf("[%v]%v\n", id, nodeTypeName(node))) + if node.InputStreams().Len() > 0 { + sb.WriteString("SIn: ") + for i, in := range node.InputStreams().Slots { + if i > 0 { + sb.WriteString(", ") + } + + if in == nil { + sb.WriteString("?") + } else { + sb.WriteString(fmt.Sprintf("%v", nodeIDs[in.Src])) + } + } + sb.WriteString("\n") + } + if node.OutputStreams().Len() > 0 { + sb.WriteString("SOut: ") + for i, out := range node.OutputStreams().Slots { + if i > 0 { + sb.WriteString(", ") + } + + sb.WriteString("(") + for i2, dst := range out.Dst { + if i2 > 0 { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("%v", nodeIDs[dst])) + } + sb.WriteString(")") + } + sb.WriteString("\n") + } + + if node.InputValues().Len() > 0 { + sb.WriteString("VIn: ") + for i, in := range node.InputValues().Slots { + if i > 0 { + sb.WriteString(", ") + } + + if in == nil { + sb.WriteString("?") + } else { + sb.WriteString(fmt.Sprintf("%v", nodeIDs[in.Src])) + } + } + sb.WriteString("\n") + } + if node.OutputValues().Len() > 0 { + sb.WriteString("VOut: ") + for i, out := range node.OutputValues().Slots { + if i > 0 { + sb.WriteString(", ") + } + + sb.WriteString("(") + for i2, dst := range out.Dst { + if i2 > 0 { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("%v", nodeIDs[dst])) + } + sb.WriteString(")") + } + sb.WriteString("\n") + } + + } + return sb.String() +} + +func nodeTypeName(node Node) string { + typ := reflect2.TypeOfValue(node) + for typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + return typ.Name() +} + func AddNode[N Node](graph *Graph, typ N) N { graph.AddNode(typ) return typ diff --git a/pkgs/ioswitch/dag/node.go b/pkgs/ioswitch/dag/node.go index 60f8ea9..df6e801 100644 --- a/pkgs/ioswitch/dag/node.go +++ b/pkgs/ioswitch/dag/node.go @@ -20,24 +20,28 @@ type NodeEnv struct { Pinned bool // 如果为true，则不应该改变这个节点的执行环境 } -func (e *NodeEnv) ToEnvUnknown() { +func (e *NodeEnv) ToEnvUnknown(pinned bool) { e.Type = EnvUnknown e.Worker = nil + e.Pinned = pinned } -func (e *NodeEnv) ToEnvDriver() { +func (e *NodeEnv) ToEnvDriver(pinned bool) { e.Type = EnvDriver e.Worker = nil + e.Pinned = pinned } -func (e *NodeEnv) ToEnvWorker(worker exec.WorkerInfo) { +func (e *NodeEnv) ToEnvWorker(worker exec.WorkerInfo, pinned bool) { e.Type = EnvWorker e.Worker = worker + e.Pinned = pinned } func (e *NodeEnv) CopyFrom(other *NodeEnv) { e.Type = other.Type e.Worker = other.Worker + e.Pinned = other.Pinned } func (e *NodeEnv) Equals(other *NodeEnv) bool { @@ -98,7 +102,7 @@ func (s *VarSlots[T]) Clear(val *T) { } func (s *VarSlots[T]) RemoveAt(idx int) { - (*s) = lo2.RemoveAt(*s, idx) + *s = lo2.RemoveAt(*s, idx) } func (s *VarSlots[T]) RemoveRange(start int, cnt int) { @@ -457,6 +461,20 @@ func (s StreamOutputSlot) Var() *StreamVar { return s.Node.OutputStreams().Get(s.Index) } +func (s StreamOutputSlot) ToSlot(slot StreamInputSlot) { + s.Var().To(slot.Node, slot.Index) +} + +// 查询所有输出的连接的输入槽位 +func (s StreamOutputSlot) ListDstSlots() []StreamInputSlot { + slots := make([]StreamInputSlot, s.Var().Dst.Len()) + myVar := s.Var() + for i, dst := range s.Var().Dst { + slots[i] = StreamInputSlot{Node: dst, Index: dst.InputStreams().IndexOf(myVar)} + } + return slots +} + type StreamInputSlot struct { Node Node Index int @@ -475,6 +493,20 @@ func (s ValueOutputSlot) Var() *ValueVar { return s.Node.OutputValues().Get(s.Index) } +func (s ValueOutputSlot) ToSlot(slot ValueInputSlot) { + s.Var().To(slot.Node, slot.Index) +} + +// 查询所有输出的连接的输入槽位 +func (s ValueOutputSlot) ListDstSlots() []ValueInputSlot { + slots := make([]ValueInputSlot, s.Var().Dst.Len()) + myVar := s.Var() + for i, dst := range s.Var().Dst { + slots[i] = ValueInputSlot{Node: dst, Index: dst.InputValues().IndexOf(myVar)} + } + return slots +} + type ValueInputSlot struct { Node Node Index int diff --git a/pkgs/ioswitch/dag/var.go b/pkgs/ioswitch/dag/var.go index 1a2f49c..217def7 100644 --- a/pkgs/ioswitch/dag/var.go +++ b/pkgs/ioswitch/dag/var.go @@ -92,28 +92,28 @@ func (s *DstList) Get(idx int) Node { } func (s *DstList) Add(n Node) int { - (*s) = append((*s), n) + *s = append(*s, n) return len(*s) - 1 } func (s *DstList) Remove(n Node) { for i, e := range *s { if e == n { - (*s) = lo2.RemoveAt((*s), i) + *s = lo2.RemoveAt(*s, i) return } } } func (s *DstList) RemoveAt(idx int) { - lo2.RemoveAt((*s), idx) + lo2.RemoveAt(*s, idx) } func (s *DstList) Resize(size int) { if s.Len() < size { - (*s) = append((*s), make([]Node, size-s.Len())...) + *s = append(*s, make([]Node, size-s.Len())...) } else if s.Len() > size { - (*s) = (*s)[:size] + *s = (*s)[:size] } } diff --git a/pkgs/ioswitch/exec/driver.go b/pkgs/ioswitch/exec/driver.go index 5c89ade..433b7c3 100644 --- a/pkgs/ioswitch/exec/driver.go +++ b/pkgs/ioswitch/exec/driver.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/go-multierror" "gitlink.org.cn/cloudream/common/pkgs/future" "gitlink.org.cn/cloudream/common/utils/io2" + "gitlink.org.cn/cloudream/common/utils/math2" ) type Driver struct { @@ -104,7 +105,7 @@ func (e *Driver) execute() { type DriverWriteStream struct { ID VarID - RangeHint *Range + RangeHint *math2.Range } type DriverReadStream struct { diff --git a/pkgs/ioswitch/exec/utils.go b/pkgs/ioswitch/exec/utils.go index 64b00c9..5186943 100644 --- a/pkgs/ioswitch/exec/utils.go +++ b/pkgs/ioswitch/exec/utils.go @@ -2,96 +2,8 @@ package exec import ( "github.com/google/uuid" - "gitlink.org.cn/cloudream/common/utils/math2" ) func genRandomPlanID() PlanID { return PlanID(uuid.NewString()) } - -type Range struct { - Offset int64 - Length *int64 -} - -func NewRange(offset int64, length int64) Range { - return Range{Offset: offset, Length: &length} -} - -func (r *Range) Extend(other Range) { - newOffset := math2.Min(r.Offset, other.Offset) - - if r.Length == nil { - r.Offset = newOffset - return - } - - if other.Length == nil { - r.Offset = newOffset - r.Length = nil - return - } - - otherEnd := other.Offset + *other.Length - rEnd := r.Offset + *r.Length - - newEnd := math2.Max(otherEnd, rEnd) - r.Offset = newOffset - *r.Length = newEnd - newOffset -} - -func (r *Range) ExtendStart(start int64) { - r.Offset = math2.Min(r.Offset, start) -} - -func (r *Range) ExtendEnd(end int64) { - if r.Length == nil { - return - } - - rEnd := r.Offset + *r.Length - newLen := math2.Max(end, rEnd) - r.Offset - r.Length = &newLen -} - -func (r *Range) Fix(maxLength int64) { - if r.Length != nil { - return - } - - len := maxLength - r.Offset - r.Length = &len -} - -func (r *Range) ToStartEnd(maxLen int64) (start int64, end int64) { - if r.Length == nil { - return r.Offset, maxLen - } - - end = r.Offset + *r.Length - return r.Offset, end -} - -func (r *Range) ClampLength(maxLen int64) { - if r.Length == nil { - return - } - - *r.Length = math2.Min(*r.Length, maxLen-r.Offset) -} - -func (r *Range) Equals(other Range) bool { - if r.Offset != other.Offset { - return false - } - - if r.Length == nil && other.Length == nil { - return true - } - - if r.Length == nil || other.Length == nil { - return false - } - - return *r.Length == *other.Length -} diff --git a/pkgs/ioswitch/plan/generate.go b/pkgs/ioswitch/plan/compile.go similarity index 96% rename from pkgs/ioswitch/plan/generate.go rename to pkgs/ioswitch/plan/compile.go index addeaed..bcdf20c 100644 --- a/pkgs/ioswitch/plan/generate.go +++ b/pkgs/ioswitch/plan/compile.go @@ -6,7 +6,7 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops" ) -func Generate(graph *dag.Graph, planBld *exec.PlanBuilder) error { +func Compile(graph *dag.Graph, planBld *exec.PlanBuilder) error { myGraph := &ops.GraphNodeBuilder{graph} generateSend(myGraph) return buildPlan(graph, planBld) @@ -42,7 +42,7 @@ func generateSend(graph *ops.GraphNodeBuilder) { dstNode := out.Dst.Get(0) getNode := graph.NewGetStream(node.Env().Worker) - getNode.Env().ToEnvDriver() + getNode.Env().ToEnvDriver(true) // // 同时需要对此变量生成HoldUntil指令，避免Plan结束时Get指令还未到达 holdNode := graph.NewHoldUntil() @@ -86,7 +86,7 @@ func generateSend(graph *ops.GraphNodeBuilder) { // // 如果是要送到Driver，则只能由Driver主动去拉取 dstNode := out.Dst.Get(0) getNode := graph.NewGetValue(node.Env().Worker) - getNode.Env().ToEnvDriver() + getNode.Env().ToEnvDriver(true) // // 同时需要对此变量生成HoldUntil指令，避免Plan结束时Get指令还未到达 holdNode := graph.NewHoldUntil() diff --git a/pkgs/ioswitch/plan/ops/driver.go b/pkgs/ioswitch/plan/ops/driver.go index da3c62f..3a50232 100644 --- a/pkgs/ioswitch/plan/ops/driver.go +++ b/pkgs/ioswitch/plan/ops/driver.go @@ -3,6 +3,7 @@ package ops import ( "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" + "gitlink.org.cn/cloudream/common/utils/math2" ) type FromDriverNode struct { @@ -40,7 +41,7 @@ func (t *FromDriverNode) GenerateOp() (exec.Op, error) { type ToDriverNode struct { dag.NodeBase Handle *exec.DriverReadStream - Range exec.Range + Range math2.Range } func (b *GraphNodeBuilder) NewToDriver(handle *exec.DriverReadStream) *ToDriverNode { @@ -57,8 +58,8 @@ func (t *ToDriverNode) SetInput(v *dag.StreamVar) { v.To(t, 0) } -func (t *ToDriverNode) Input() dag.StreamOutputSlot { - return dag.StreamOutputSlot{ +func (t *ToDriverNode) Input() dag.StreamInputSlot { + return dag.StreamInputSlot{ Node: t, Index: 0, } diff --git a/pkgs/ioswitch/plan/ops/store.go b/pkgs/ioswitch/plan/ops/store.go index 977af42..caf45e2 100644 --- a/pkgs/ioswitch/plan/ops/store.go +++ b/pkgs/ioswitch/plan/ops/store.go @@ -23,7 +23,21 @@ func (o *Store) Execute(ctx *exec.ExecContext, e *exec.Executor) error { } func (o *Store) String() string { - return fmt.Sprintf("Store %v: %v", o.Key, o.Var) + return fmt.Sprintf("Store %v as \"%v\"", o.Var, o.Key) +} + +type StoreConst struct { + Key string + Value exec.VarValue +} + +func (o *StoreConst) Execute(ctx *exec.ExecContext, e *exec.Executor) error { + e.Store(o.Key, o.Value) + return nil +} + +func (o *StoreConst) String() string { + return fmt.Sprintf("StoreConst %v: %v", o.Key, o.Value) } type StoreNode struct { @@ -53,3 +67,28 @@ func (t *StoreNode) GenerateOp() (exec.Op, error) { // func (t *StoreType) String() string { // return fmt.Sprintf("Store[%s]%v%v", t.StoreKey, formatStreamIO(node), formatValueIO(node)) // } + +type StoreConstNode struct { + dag.NodeBase + Key string + Value exec.VarValue +} + +func (b *GraphNodeBuilder) NewStoreConst(key string, value exec.VarValue) *StoreConstNode { + node := &StoreConstNode{ + Key: key, + Value: value, + } + b.AddNode(node) + return node +} + +func (t *StoreConstNode) GenerateOp() (exec.Op, error) { + return &StoreConst{ + Key: t.Key, + Value: t.Value, + }, nil +} + +// func (t *StoreConstType) String() string { +// return fmt.Sprintf("StoreConst[%s]%v%v", t.StoreKey, formatStreamIO(node), formatValueIO(node)) diff --git a/pkgs/ioswitch/plan/ops/sync.go b/pkgs/ioswitch/plan/ops/sync.go index c4ec53c..3e096cb 100644 --- a/pkgs/ioswitch/plan/ops/sync.go +++ b/pkgs/ioswitch/plan/ops/sync.go @@ -113,7 +113,7 @@ func (w *HoldUntil) Execute(ctx *exec.ExecContext, e *exec.Executor) error { } func (w *HoldUntil) String() string { - return fmt.Sprintf("HoldUntil Waits: %v, (%v) -> (%v)", utils.FormatVarIDs(w.Waits), utils.FormatVarIDs(w.Holds), utils.FormatVarIDs(w.Emits)) + return fmt.Sprintf("HoldUntil(waits=%v): %v -> %v", utils.FormatVarIDs(w.Waits), utils.FormatVarIDs(w.Holds), utils.FormatVarIDs(w.Emits)) } type HangUntil struct { diff --git a/pkgs/ioswitch/plan/ops/var.go b/pkgs/ioswitch/plan/ops/var.go index 962ec4c..973cc59 100644 --- a/pkgs/ioswitch/plan/ops/var.go +++ b/pkgs/ioswitch/plan/ops/var.go @@ -1,6 +1,7 @@ package ops import ( + "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" ) @@ -21,3 +22,31 @@ func (o *ConstVar) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ConstVar) String() string { return "ConstVar" } + +type ConstNode struct { + dag.NodeBase + Value exec.VarValue +} + +func (b *GraphNodeBuilder) NewConst(val exec.VarValue) *ConstNode { + node := &ConstNode{ + Value: val, + } + b.AddNode(node) + + node.OutputValues().Init(node, 1) + return node +} + +func (t *ConstNode) Output() dag.ValueOutputSlot { + return dag.ValueOutputSlot{ + Node: t, + Index: 0, + } +} + +func (t *ConstNode) GenerateOp() (exec.Op, error) { + return &DropStream{ + Input: t.Output().Var().VarID, + }, nil +} diff --git a/pkgs/logger/global_logger.go b/pkgs/logger/global_logger.go index 1a577cc..6d07e73 100644 --- a/pkgs/logger/global_logger.go +++ b/pkgs/logger/global_logger.go @@ -83,6 +83,14 @@ func Init(cfg *Config) error { // 下面是日志记录的方法，它们分别对应不同的日志级别和格式。 // 这些方法最终都会调用logrus对应的方法来记录日志。 +func Trace(args ...interface{}) { + logrus.Trace(args...) +} + +func Tracef(format string, args ...interface{}) { + logrus.Tracef(format, args...) +} + func Debug(args ...interface{}) { logrus.Debug(args...) } diff --git a/pkgs/logger/logger.go b/pkgs/logger/logger.go index 3e0a52b..ad67986 100644 --- a/pkgs/logger/logger.go +++ b/pkgs/logger/logger.go @@ -30,6 +30,9 @@ var loggerLevels = map[string]logrus.Level{ } type Logger interface { + Trace(args ...interface{}) + Tracef(format string, args ...interface{}) + Debug(args ...interface{}) Debugf(format string, args ...interface{}) diff --git a/pkgs/logger/logrus_logger.go b/pkgs/logger/logrus_logger.go index a69eda3..16ba253 100644 --- a/pkgs/logger/logrus_logger.go +++ b/pkgs/logger/logrus_logger.go @@ -10,6 +10,13 @@ type logrusLogger struct { entry *logrus.Entry } +func (l *logrusLogger) Trace(args ...interface{}) { + l.entry.Trace(args...) +} +func (l *logrusLogger) Tracef(format string, args ...interface{}) { + l.entry.Tracef(format, args...) +} + func (l *logrusLogger) Debug(args ...interface{}) { l.entry.Debug(args...) } diff --git a/pkgs/tickevent/executor.go b/pkgs/tickevent/executor.go index 0bb0a22..f960404 100644 --- a/pkgs/tickevent/executor.go +++ b/pkgs/tickevent/executor.go @@ -20,7 +20,7 @@ type EventTicker[TArgs any] struct { event TickEvent[TArgs] intervalMs int doneChan chan int - done atomic.Bool + done *atomic.Bool } type Executor[TArgs any] struct { @@ -43,7 +43,7 @@ func (e *Executor[TArgs]) Start(event TickEvent[TArgs], intervalMs int, opts ... event: event, intervalMs: intervalMs, doneChan: make(chan int), - done: atomic.Bool{}, + done: &atomic.Bool{}, } ticker.done.Store(false) diff --git a/pkgs/trie/trie.go b/pkgs/trie/trie.go index 829a49f..18e340c 100644 --- a/pkgs/trie/trie.go +++ b/pkgs/trie/trie.go @@ -102,10 +102,15 @@ func (n *Node[T]) RemoveSelf(cleanParent bool) { } // 修改时需要注意允许在visitorFn中删除当前节点 -func (n *Node[T]) Iterate(visitorFn func(word string, node *Node[T], isWordNode bool) VisitCtrl) { +func (n *Node[T]) Iterate(visitorFn func(path []string, node *Node[T], isWordNode bool) VisitCtrl) { + n.iterate([]string{}, visitorFn) +} + +func (n *Node[T]) iterate(path []string, visitorFn func(path []string, node *Node[T], isWordNode bool) VisitCtrl) { if n.WordNexts != nil { for word, node := range n.WordNexts { - ret := visitorFn(word, node, true) + newPath := append(path, word) + ret := visitorFn(newPath, node, true) if ret == VisitBreak { return } @@ -114,12 +119,13 @@ func (n *Node[T]) Iterate(visitorFn func(word string, node *Node[T], isWordNode continue } - node.Iterate(visitorFn) + node.iterate(newPath, visitorFn) } } if n.AnyNext != nil { - ret := visitorFn("", n.AnyNext, false) + newPath := append(path, "") + ret := visitorFn(newPath, n.AnyNext, false) if ret == VisitBreak { return } @@ -128,7 +134,7 @@ func (n *Node[T]) Iterate(visitorFn func(word string, node *Node[T], isWordNode return } - n.AnyNext.Iterate(visitorFn) + n.AnyNext.iterate(newPath, visitorFn) } } @@ -160,10 +166,11 @@ func (t *Trie[T]) WalkEnd(words []string) (*Node[T], bool) { ptr := &t.Root for _, word := range words { - ptr = ptr.WalkNext(word) - if ptr == nil { - return nil, false + np := ptr.WalkNext(word) + if np == nil { + return ptr, false } + ptr = np } return ptr, true @@ -198,6 +205,6 @@ func (t *Trie[T]) CreateWords(words []string) *Node[T] { return ptr } -func (n *Trie[T]) Iterate(visitorFn func(word string, node *Node[T], isWordNode bool) VisitCtrl) { +func (n *Trie[T]) Iterate(visitorFn func(path []string, node *Node[T], isWordNode bool) VisitCtrl) { n.Root.Iterate(visitorFn) } diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go new file mode 100644 index 0000000..c905db1 --- /dev/null +++ b/sdks/blockchain/blockchain.go @@ -0,0 +1,146 @@ +package blockchain + +import ( + "fmt" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" + "time" +) + +type InvokeReq struct { + ContractAddress string `json:"contractAddress"` + FunctionName string `json:"functionName"` + MemberName string `json:"memberName"` + Type string `json:"type"` + Args []string `json:"args"` + Amount int64 `json:"amount"` +} + +func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { + targetUrl, err := url.JoinPath(c.baseURL, "/jcc-bcos/contract/invoke") + if err != nil { + return err + } + + header := make(map[string]string) + header["Content-Type"] = http2.ContentTypeJSON + header["Authorization"] = token + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: header, + }) + if err != nil { + println(err) + return err + } + + var codeResp response[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + println(err) + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code != ResponseCodeOK { + println(codeResp.ToError().Message) + return codeResp.ToError() + } + + return nil +} + +type UnPledgePointsReq struct { + PrivateKey string `form:"privateKey"` + BusinessCode string `form:"businessCode"` + EndTime time.Time `form:"endTime"` +} + +func (c *Client) UnPledgePoints(req UnPledgePointsReq, token string) error { + targetUrl, err := url.JoinPath(c.baseURL, "/jcc-bcos/points/unPledgePoints") + if err != nil { + return err + } + + header := make(map[string]string) + header["Content-Type"] = http2.ContentTypeJSON + header["Authorization"] = token + + resp, err := http2.PutJSON(targetUrl, http2.RequestParam{ + Query: req, + Header: header, + }) + if err != nil { + println(err) + return err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == 200 { + return nil + } + + return codeResp.ToError() + } + + return nil +} + +type TokenReq struct { + Username string `json:"username"` + Password string `json:"password"` +} + +type TokenResp struct { + RoleID string `json:"roleId"` + Token string `json:"token"` +} + +func (c *Client) getToken() (string, error) { + targetUrl, err := url.JoinPath(c.loginUrl, "/jcc-admin/admin/login") + if err != nil { + return "", err + } + + req := TokenReq{ + Username: c.userName, + Password: c.password, + } + + header := make(map[string]string) + header["User-Agent"] = "Apifox/1.0.0 (https://apifox.com)" + header["Content-Type"] = "application/json" + header["Accept"] = "*/*" + header["Host"] = "dev.jointcloud.net" + header["Connection"] = "keep-alive" + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: header, + }) + if err != nil { + return "", err + } + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + + var codeResp response[TokenResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return "", fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return "Bearer " + codeResp.Data.Token, nil + } + + return "", codeResp.ToError() + } + return "", fmt.Errorf("error: %w", err) +} diff --git a/sdks/blockchain/client.go b/sdks/blockchain/client.go new file mode 100644 index 0000000..7b4b5af --- /dev/null +++ b/sdks/blockchain/client.go @@ -0,0 +1,63 @@ +package blockchain + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string + loginUrl string + userName string + password string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + loginUrl: cfg.LoginUrl, + userName: cfg.UserName, + password: cfg.Password, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/blockchain/config.go b/sdks/blockchain/config.go new file mode 100644 index 0000000..a3e02f4 --- /dev/null +++ b/sdks/blockchain/config.go @@ -0,0 +1,12 @@ +package blockchain + +type Config struct { + URL string `json:"url"` + ContractAddress string `json:"contractAddress"` + FunctionName string `json:"functionName"` + LoginUrl string `json:"loginUrl"` + UserName string `json:"userName"` + Password string `json:"password"` + //MemberName string `json:"memberName"` + //Type string `json:"type"` +} diff --git a/sdks/blockchain/models.go b/sdks/blockchain/models.go new file mode 100644 index 0000000..8da50d0 --- /dev/null +++ b/sdks/blockchain/models.go @@ -0,0 +1,3 @@ +package blockchain + +type ClusterID string diff --git a/sdks/blockchain/test.go b/sdks/blockchain/test.go new file mode 100644 index 0000000..8a85711 --- /dev/null +++ b/sdks/blockchain/test.go @@ -0,0 +1,37 @@ +package blockchain + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" +) + +func main() { + url := "http://localhost:2006/contract/invoke" + method := "POST" + payload := strings.NewReader(`{` + " " + ` "contractAddress" : "0xc860ab27901b3c2b810165a6096c64d88763617f",` + " " + ` "functionName" : "storeEvidence",` + " " + ` "args" : ["3","touteng"],` + " " + ` "memberName" :"pcm",` + " " + ` "type": "2"` + " " + ` }`) + client := &http.Client{} + req, err := http.NewRequest(method, url, payload) + if err != nil { + fmt.Println(err) + return + } + req.Header.Add("User-Agent", "Apifox/1.0.0 (https://apifox.com)") + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "*/*") + req.Header.Add("Host", "localhost:2006") + req.Header.Add("Connection", "keep-alive") + res, err := client.Do(req) + if err != nil { + fmt.Println(err) + return + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + return + } + fmt.Println(string(body)) +} diff --git a/sdks/hpc/client.go b/sdks/hpc/client.go new file mode 100644 index 0000000..07b4393 --- /dev/null +++ b/sdks/hpc/client.go @@ -0,0 +1,63 @@ +package hpc + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type respons2[T any] struct { + Code int `json:"code"` + Message string `json:"msg"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/hpc/config.go b/sdks/hpc/config.go new file mode 100644 index 0000000..d168c41 --- /dev/null +++ b/sdks/hpc/config.go @@ -0,0 +1,5 @@ +package hpc + +type Config struct { + URL string `json:"url"` +} diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go new file mode 100644 index 0000000..8c9acad --- /dev/null +++ b/sdks/hpc/job.go @@ -0,0 +1,83 @@ +package hpc + +import ( + "fmt" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" +) + +type CreateHPCJobReq struct { + Name string `json:"name"` + Description string `json:"description"` + ClusterID schsdk.ClusterID `json:"clusterId"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + ScriptContent string `json:"scriptContent"` + //Parameters HPCParameter `json:"parameters"` + Parameters map[string]string `json:"parameters"` +} + +//type HPCParameter struct { +// JobName string `json:"jobName"` +// Partition string `json:"partition"` +// Ntasks string `json:"ntasks"` +// Nodes string `json:"nodes"` +// BamFile string `json:"bamFile"` +// InputFile string `json:"inputFile"` +//} + +type CreateJobResp struct { + Backend string `json:"backend"` + JobInfo HPCJobInfo `json:"jobInfo"` +} + +type HPCJobInfo struct { + JobDir string `json:"jobDir"` + JobID string `json:"jobId"` + TaskID string `json:"taskId"` +} + +func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/hpc/commitHpcTask") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + // 打印resp.Body内容 + //body, err := io.ReadAll(resp.Body) + //if err != nil { + // println("Error reading response body:", err) + //} + //println("Response Body:", string(body)) + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, fmt.Errorf("error: %s", codeResp.Message) + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) + +} diff --git a/sdks/hpc/models.go b/sdks/hpc/models.go new file mode 100644 index 0000000..cee9586 --- /dev/null +++ b/sdks/hpc/models.go @@ -0,0 +1,525 @@ +package hpc + +import ( + "gitlink.org.cn/cloudream/common/pkgs/types" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/serder" + "time" +) + +type ResourceType string + +const ( + ResourceTypeCPU ResourceType = "CPU" + ResourceTypeNPU ResourceType = "NPU" + ResourceTypeGPU ResourceType = "GPU" + ResourceTypeMLU ResourceType = "MLU" + ResourceTypeStorage ResourceType = "STORAGE" + ResourceTypeMemory ResourceType = "MEMORY" + + Split = "/" + + CODE = "code" + DATASET = "dataset" + IMAGE = "image" + MODEL = "model" + RESULT = "result" + + OrderByName = "name" + OrderBySize = "size" + OrderByTime = "time" + + StorageTypeURL = "url" + StorageTypeJCS = "jcs" + + RejectedStatus = "rejected" + PendingStatus = "pending" + ApprovedStatus = "approved" + RevokedStatus = "revoked" + CancelStatus = "cancel" + ExpiredStatus = "expired" + + ApplyAccess = "apply" + PrivateAccess = "private" + PublicAccess = "public" + + PreferencePriority = "preference" + SpecifyClusterPriority = "specify" + + FailedStatus = "failed" + SuccessStatus = "success" + + Query = "query" + Delete = "delete" + + ChildrenType = "children" + ParentType = "parent" + + PlatformSugon = "sugon" + PlatformOpenI = "OpenI" + PlatformModelArts = "ModelArts" + + URL = "url" + ID = "id" + + Startup = "startup" +) + +type TaskID int64 +type DataID int64 + +type ClusterDetail struct { + // 集群ID + ClusterId schsdk.ClusterID `json:"clusterID"` + // 集群功能类型：云算，智算，超算 + ClusterType string `json:"clusterType"` + // 集群地区：华东地区、华南地区、华北地区、华中地区、西南地区、西北地区、东北地区 + Region string `json:"region"` + // 资源类型 + Resources2 []ResourceData `json:"resources1,omitempty"` + //Resources2 []ResourceData `json:"resources"` + Resources []ClusterResource `json:"resources"` +} + +type ClusterResource struct { + Resource TmpResourceData `json:"resource"` + BaseResources []TmpResourceData `json:"baseResources"` +} + +type TmpResourceData struct { + Type ResourceType `json:"type"` + Name string `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceData interface { + Noop() +} + +var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( + (*CPUResourceData)(nil), + (*NPUResourceData)(nil), + (*GPUResourceData)(nil), + (*MLUResourceData)(nil), + (*DCUResourceData)(nil), + (*GCUResourceData)(nil), + (*GPGPUResourceData)(nil), + (*StorageResourceData)(nil), + (*MemoryResourceData)(nil), + (*BalanceResourceData)(nil), + (*RateResourceData)(nil), +) +var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "type") + +type ResourceDataBase struct{} + +func (d *ResourceDataBase) Noop() {} + +type UnitValue[T any] struct { + Unit string `json:"unit"` + Value T `json:"value"` +} + +type CPUResourceData struct { + serder.Metadata `union:"CPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type NPUResourceData struct { + serder.Metadata `union:"NPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPUResourceData struct { + serder.Metadata `union:"GPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type MLUResourceData struct { + serder.Metadata `union:"MLU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type DCUResourceData struct { + serder.Metadata `union:"DCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GCUResourceData struct { + serder.Metadata `union:"GCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPGPUResourceData struct { + serder.Metadata `union:"ILUVATAR-GPGPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type StorageResourceData struct { + serder.Metadata `union:"STORAGE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type MemoryResourceData struct { + serder.Metadata `union:"MEMORY"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type BalanceResourceData struct { + serder.Metadata `union:"BALANCE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type RateResourceData struct { + serder.Metadata `union:"RATE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceRange struct { + UserID cdssdk.UserID `json:"userID"` + Type ResourceType `json:"type"` + GPU Range `json:"gpu"` + GPUNumber int `json:"gpuNumber"` + CPU Range `json:"cpu"` + Memory Range `json:"memory"` + Storage Range `json:"storage"` +} + +type Range struct { + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +type ResourcePriority interface { + Noop() +} + +type ResourcePriorityBase struct { +} + +var ResourcePriorityTypeUnion = types.NewTypeUnion[ResourcePriority]( + (*RegionPriority)(nil), + (*ChipPriority)(nil), + (*BiasPriority)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&ResourcePriorityTypeUnion, "type") + +func (d *ResourcePriorityBase) Noop() {} + +type RegionPriority struct { + serder.Metadata `union:"region"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type ChipPriority struct { + serder.Metadata `union:"chip"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type BiasPriority struct { + serder.Metadata `union:"bias"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type TaskMessage struct { + Status string `json:"status"` + Message string `json:"message"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` +} + +type UploadParams struct { + DataType string `json:"dataType"` + UploadInfo UploadInfo `json:"uploadInfo"` +} + +type UploadInfo interface { + Noop() +} + +var UploadInfoTypeUnion = types.NewTypeUnion[UploadInfo]( + (*LocalUploadInfo)(nil), + (*RemoteUploadInfo)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type") + +type LocalUploadInfo struct { + serder.Metadata `union:"local"` + UploadInfoBase + Type string `json:"type"` + LocalPath string `json:"localPath"` + ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` +} + +type RemoteUploadInfo struct { + serder.Metadata `union:"url"` + UploadInfoBase + Type string `json:"type"` + Url string `json:"url"` + Branch string `json:"branch"` + DataName string `json:"dataName"` + Cluster schsdk.ClusterID `json:"clusterID"` +} + +type UploadInfoBase struct{} + +func (d *UploadInfoBase) Noop() {} + +type UploadPriority interface { + Noop() +} + +var UploadPriorityTypeUnion = types.NewTypeUnion[UploadPriority]( + (*Preferences)(nil), + (*SpecifyCluster)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadPriorityTypeUnion, "type") + +type Preferences struct { + serder.Metadata `union:"preference"` + UploadPriorityBase + Type string `json:"type"` + ResourcePriorities []ResourcePriority `json:"priorities"` +} + +type SpecifyCluster struct { + serder.Metadata `union:"specify"` + UploadPriorityBase + Type string `json:"type"` + Clusters []schsdk.ClusterID `json:"clusters"` +} + +type UploadPriorityBase struct{} + +func (d *UploadPriorityBase) Noop() {} + +type QueryData struct { + DataType string `json:"dataType" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Path string `json:"path"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + CurrentPage int `json:"currentPage" binding:"required"` + PageSize int `json:"pageSize" binding:"required"` + OrderBy string `json:"orderBy" binding:"required"` +} + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*DatasetBinding)(nil), + (*ModelBinding)(nil), + (*CodeBinding)(nil), + (*ImageBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct{} + +func (d *DataBindingBase) Noop() {} + +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints float64 `json:"points"` +} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` +} + +type CodeBinding struct { + serder.Metadata `union:"code"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Description string `json:"description"` + ImageID schsdk.ImageID `json:"imageID"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` + // 当集群为openi的时候，需要传入分支 + Branch string `json:"branch"` +} + +//type ImageBinding struct { +// serder.Metadata `union:"image"` +// DataBindingBase +// Type string `json:"type"` +// Name string `json:"name"` +// ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` +// Description string `json:"description"` +// Architecture string `json:"architecture"` +// ResourceType string `json:"resourceType"` +// Tags []string `json:"tags"` +// PackageID cdssdk.PackageID `json:"packageID"` +//} + +type ImageBinding struct { + serder.Metadata `union:"image"` + DataBindingBase + Type string `json:"type"` + ID int64 `json:"id"` + Name string `json:"name"` + IDType string `json:"idType"` + ImageID string `json:"imageID"` + ClusterID schsdk.ClusterID `json:"clusterID"` +} + +type Image struct { + ImageID schsdk.ImageID `json:"imageID" gorm:"column:ImageID;primaryKey"` + Name string `json:"name" gorm:"column:Name"` + CreateTime time.Time `json:"createTime" gorm:"column:CreateTime"` + ClusterImage []ClusterImage `gorm:"foreignKey:image_id;references:ImageID" json:"clusterImages"` +} + +type ClusterImage struct { + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + OriginImageType string `gorm:"column:origin_image_type" json:"originImageType"` + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + OriginImageName string `gorm:"column:origin_image_name" json:"originImageName"` + ClusterImageCard []ClusterImageCard `gorm:"foreignKey:origin_image_id;references:origin_image_id" json:"cards"` +} + +func (ClusterImage) TableName() string { + return "cluster_image" +} + +type ClusterImageCard struct { + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + Card string `gorm:"column:card" json:"card"` +} + +func (ClusterImageCard) TableName() string { + return "cluster_image_card" +} + +type QueryBindingFilters struct { + Status string `json:"status"` + Name string `json:"name"` +} + +type QueryBindingDataParam interface { + Noop() +} + +var QueryBindingDataParamTypeUnion = types.NewTypeUnion[QueryBindingDataParam]( + (*PrivateLevel)(nil), + (*ApplyLevel)(nil), + (*PublicLevel)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&QueryBindingDataParamTypeUnion, "type") + +type QueryBindingDataParamBase struct{} + +func (d *QueryBindingDataParamBase) Noop() {} + +type PrivateLevel struct { + serder.Metadata `union:"private"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + BindingID int64 `json:"bindingID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type ApplyLevel struct { + serder.Metadata `union:"apply"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type PublicLevel struct { + serder.Metadata `union:"public"` + QueryBindingDataParamBase + UserID cdssdk.UserID `json:"userID" binding:"required"` + Type string `json:"type" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} diff --git a/sdks/pcmscheduler/access.go b/sdks/pcmscheduler/access.go new file mode 100644 index 0000000..82b1dc2 --- /dev/null +++ b/sdks/pcmscheduler/access.go @@ -0,0 +1,54 @@ +package sch + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) + +type AccessRequest struct { + ID uint `gorm:"column:id" json:"ID"` + BindingID int64 `gorm:"column:binding_id" json:"packageId"` + ApplicantID int64 `gorm:"column:applicant_id" json:"applicantID"` + Reason string `gorm:"column:reason" json:"reason"` + ExpirationDate time.Time `gorm:"column:expiration_date" json:"expirationDate"` + Status string `gorm:"column:status" json:"status"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` +} + +type PermissionApply struct { + BindingID int64 `gorm:"column:binding_id" json:"bindingID"` + ApplicantID cdssdk.UserID `gorm:"column:applicant_id" json:"applicantID"` // 申请人ID + OwnerID cdssdk.UserID `gorm:"column:data_owner_id" json:"ownerID"` // 数据拥有者ID + Reason string `gorm:"column:apply_reason" json:"reason"` // 申请理由 + ExpirationDate time.Time `gorm:"column:expiration_date" json:"expirationDate"` // 过期时间 + Status string `gorm:"column:status" json:"status"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` +} + +type PermissionApproval struct { + ID int64 `json:"id"` + Reason string `gorm:"column:reject_reason" json:"reason"` + Status string `gorm:"column:status" json:"status"` +} + +type AccessLog struct { + //ID uint `json:"id"` + BindingID int64 `gorm:"column:binding_id" json:"packageId"` + ApplicantID cdssdk.UserID `gorm:"column:applicant_id" json:"applicantID"` + OwnerID cdssdk.UserID `gorm:"column:data_owner_id" json:"OwnerID"` + AccessTime time.Time `gorm:"column:access_time" json:"accessTime"` + AccessType string `gorm:"column:access_type" json:"accessType"` +} + +type User struct { + ID int64 `gorm:"column:id" json:"id"` + SsoID string `gorm:"column:sso_id" json:"ssoID"` + UserName string `gorm:"column:username" json:"userName"` + Created time.Time `gorm:"column:created_at" json:"created"` +} + +type Bucket struct { + ID cdssdk.BucketID `gorm:"column:id" json:"id"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + DataType string `gorm:"column:data_type" json:"dataType"` +} diff --git a/sdks/pcmscheduler/client.go b/sdks/pcmscheduler/client.go new file mode 100644 index 0000000..fde07fc --- /dev/null +++ b/sdks/pcmscheduler/client.go @@ -0,0 +1,63 @@ +package sch + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type respons2[T any] struct { + Code int `json:"code"` + Message string `json:"msg"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/pcmscheduler/config.go b/sdks/pcmscheduler/config.go new file mode 100644 index 0000000..07b0093 --- /dev/null +++ b/sdks/pcmscheduler/config.go @@ -0,0 +1,5 @@ +package sch + +type Config struct { + URL string `json:"url"` +} diff --git a/sdks/pcmscheduler/job.go b/sdks/pcmscheduler/job.go new file mode 100644 index 0000000..3a5d291 --- /dev/null +++ b/sdks/pcmscheduler/job.go @@ -0,0 +1,24 @@ +package sch + +import ( + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) + +type PCMJob struct { + ID string `gorm:"column:id" json:"ID"` + JobType string `gorm:"column:job_type" json:"jobType"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"` + LocalJobID string `gorm:"column:local_job_id" json:"localJobID"` + Param string `gorm:"column:param" json:"param"` + Token string `gorm:"column:token" json:"token"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` +} + +type PCMJobDataReturn struct { + JobID string `gorm:"column:job_id" json:"jobID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go new file mode 100644 index 0000000..bcf329c --- /dev/null +++ b/sdks/pcmscheduler/jobset.go @@ -0,0 +1,314 @@ +package sch + +import ( + "fmt" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" +) + +type GetClusterInfoReq struct { + IDs []schsdk.ClusterID `json:"clusterIDs"` +} + +func (c *Client) GetClusterInfo(req GetClusterInfoReq, token string) ([]ClusterDetail, error) { + targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + + var codeResp response[[]ClusterDetail] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + // 该接口传入参数后查询效率很低，所以需要在这里特殊处理 + if codeResp.Code == ResponseCodeOK { + var results []ClusterDetail + for _, cluster := range codeResp.Data { + for _, id := range req.IDs { + if cluster.ClusterId == id { + results = append(results, cluster) + } + } + } + return results, nil + } + + return nil, codeResp.ToError() + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} + +type CreateInferenceJobResp struct { + TaskId string `json:"taskId"` +} + +type CreateAIJobReq struct { + Name string `json:"name"` + Description string `json:"description"` + JobResources schsdk.JobResources `json:"jobResources"` + DataDistributes DataDistribute `json:"dataDistributes"` +} + +type CommonJsonData struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type DataDistribute struct { + Dataset []DatasetDistribute `json:"dataset"` + Code []CodeDistribute `json:"code"` + Image []ImageDistribute `json:"image"` + Model []ModelDistribute `json:"model"` +} + +type DataDetail struct { + ClusterID schsdk.ClusterID `json:"clusterID"` + //StorageID cdssdk.StorageID `json:"storageID"` + StorageID cdssdk.StorageID + JsonData string `json:"jsonData"` +} + +type DatasetDistribute struct { + DataName string `json:"dataName"` + PackageID cdssdk.PackageID `json:"packageID"` + Clusters []DataDetail `json:"clusters"` +} + +type CodeDistribute struct { + DataName string `json:"dataName"` + PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` + Clusters []DataDetail `json:"clusters"` +} + +type ImageDistribute struct { + DataName string `json:"dataName"` + //PackageID cdssdk.PackageID `json:"packageID"` + ImageID schsdk.ImageID `json:"packageID"` + Clusters []DataDetail `json:"clusters"` +} + +type ModelDistribute struct { + DataName string `json:"dataName"` + PackageID cdssdk.PackageID `json:"packageID"` + Clusters []DataDetail `json:"clusters"` +} + +type CreateJobResp struct { + TaskID TaskID `json:"taskID"` + TaskName string `json:"taskName"` + ScheduleDatas []ScheduleData `json:"scheduleDatas"` +} + +type ScheduleData struct { + DataType string `json:"dataType"` + PackageID cdssdk.PackageID `json:"packageID"` + StorageType string `json:"storageType"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` +} + +func (c *Client) CreateInferenceJob(req CreateAIJobReq, token string) (*CreateInferenceJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateInferenceJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, fmt.Errorf("error: %s", codeResp.Message) + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} + +type StopInferenceJobReq struct { + TaskId string `json:"taskId"` +} + +func (c *Client) StopInferenceJob(req StopInferenceJobReq, token string) error { + targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") + if err != nil { + return err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateInferenceJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return nil + } + + return fmt.Errorf("error: %s", codeResp.Message) + } + + return fmt.Errorf("unknow response content type: %s", contType) +} + +func (c *Client) CreateJob(req CreateAIJobReq, token string) (*CreateJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "schedule/createTask") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, fmt.Errorf("error: %s", codeResp.Message) + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) + +} + +type RunJobReq struct { + TaskID TaskID `json:"taskID"` + ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` +} + +type DataScheduleResult struct { + Clusters []DataDetail `json:"clusters"` + PackageID cdssdk.PackageID `json:"packageID"` + PackageFullPath string `json:"packageFullPath"` + Status bool `json:"status"` + Msg string `json:"msg"` +} + +type DataScheduleResults struct { + DataType string `json:"dataType"` + Results []DataScheduleResult `json:"results"` +} + +func (c *Client) RunJob(req RunJobReq, token string) error { + targetUrl, err := url.JoinPath(c.baseURL, "schedule/runTask") + if err != nil { + return err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return nil + } + + return fmt.Errorf("error: %s", codeResp.Message) + } + + return fmt.Errorf("unknow response content type: %s", contType) + +} + +type CancelJobReq struct { + TaskID TaskID `json:"taskID"` + Msg string `json:"msg"` +} + +func (c *Client) CancelJob(req CancelJobReq) error { + targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") + if err != nil { + return err + } + resp, err := http2.GetJSON(targetUrl, http2.RequestParam{Body: req}) + if err != nil { + return err + } + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + + var codeResp response[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return nil + } + + return codeResp.ToError() + } + + return fmt.Errorf("unknow response content type: %s", contType) +} diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go new file mode 100644 index 0000000..6f8707f --- /dev/null +++ b/sdks/pcmscheduler/models.go @@ -0,0 +1,603 @@ +package sch + +import ( + "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/sdks/hpc" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/serder" + "time" +) + +type ResourceType string + +const ( + ResourceTypeCPU ResourceType = "CPU" + ResourceTypeNPU ResourceType = "NPU" + ResourceTypeGPU ResourceType = "GPU" + ResourceTypeMLU ResourceType = "MLU" + ResourceTypeStorage ResourceType = "STORAGE" + ResourceTypeMemory ResourceType = "MEMORY" + ResourceTypeDCU ResourceType = "DCU" + ResourceTypeGCU ResourceType = "GCU" + + Split = "/" + + CODE = "code" + DATASET = "dataset" + IMAGE = "image" + MODEL = "model" + RESULT = "result" + + PackageTypeNormal = "normal" + PackageTypeNull = "null" + Null = "null" + + OrderByName = "name" + OrderBySize = "size" + OrderByTime = "time" + + StorageTypeURL = "url" + StorageTypeJCS = "jcs" + + RejectedStatus = "rejected" + PendingStatus = "pending" + ApprovedStatus = "approved" + RevokedStatus = "revoked" + CancelStatus = "cancel" + ExpiredStatus = "expired" + + ApplyAccess = "apply" + PrivateAccess = "private" + PublicAccess = "public" + + PreferencePriority = "preference" + SpecifyClusterPriority = "specify" + + FailedStatus = "failed" + SuccessStatus = "success" + SucceededStatus = "succeeded" + UploadingStatus = "uploading" + RunningStatus = "running" + + Query = "query" + Delete = "delete" + + ChildrenType = "children" + ParentType = "parent" + + PlatformSugon = "sugon" + PlatformOpenI = "OpenI" + PlatformModelArts = "ModelArts" + + PlatformAI = "AI" // 智算 + PlatformCloud = "CLOUD" // 云算 + PlatformCloudInference = "PCM_Inference" + PlatformHPC = "HPCSlurm" //超算 + + URL = "url" + ID = "id" + + Startup = "startup" + + Schedule = "schedule" + + BlockChainJobCreatePrefix = "job_create_" + + Complete = "Complete" + + NodeTypeBinding = "binding" + NodeTypeUpload = "upload" + NodeTypeDataReturn = "data_return" + NodeTypeHPCCreate = "hpc_create" + NodeTypeInference = "inference" + NodeTypeAICreate = "ai_job_create" + NodeTypeAIJobRun = "ai_job_run" +) + +type TaskID int64 +type DataID int64 + +type ClusterDetail struct { + // 集群ID + ClusterId schsdk.ClusterID `json:"clusterID"` + // 集群功能类型：云算，智算，超算 + ClusterType string `json:"clusterType"` + // 集群地区：华东地区、华南地区、华北地区、华中地区、西南地区、西北地区、东北地区 + Region string `json:"region"` + // 资源类型 + Resources2 []ResourceData `json:"resources1,omitempty"` + //Resources2 []ResourceData `json:"resources"` + Resources []ClusterResource `json:"resources"` +} + +type ClusterResource struct { + Resource TmpResourceData `json:"resource"` + BaseResources []TmpResourceData `json:"baseResources"` +} + +type TmpResourceData struct { + Type ResourceType `json:"type"` + Name string `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceData interface { + Noop() +} + +var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( + (*CPUResourceData)(nil), + (*NPUResourceData)(nil), + (*GPUResourceData)(nil), + (*MLUResourceData)(nil), + (*DCUResourceData)(nil), + (*GCUResourceData)(nil), + (*GPGPUResourceData)(nil), + (*StorageResourceData)(nil), + (*MemoryResourceData)(nil), + (*BalanceResourceData)(nil), + (*RateResourceData)(nil), +) +var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "type") + +type ResourceDataBase struct{} + +func (d *ResourceDataBase) Noop() {} + +type UnitValue[T any] struct { + Unit string `json:"unit"` + Value T `json:"value"` +} + +type CPUResourceData struct { + serder.Metadata `union:"CPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type NPUResourceData struct { + serder.Metadata `union:"NPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPUResourceData struct { + serder.Metadata `union:"GPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type MLUResourceData struct { + serder.Metadata `union:"MLU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type DCUResourceData struct { + serder.Metadata `union:"DCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GCUResourceData struct { + serder.Metadata `union:"GCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPGPUResourceData struct { + serder.Metadata `union:"ILUVATAR-GPGPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type StorageResourceData struct { + serder.Metadata `union:"STORAGE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type MemoryResourceData struct { + serder.Metadata `union:"MEMORY"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type BalanceResourceData struct { + serder.Metadata `union:"BALANCE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type RateResourceData struct { + serder.Metadata `union:"RATE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceRange struct { + UserID cdssdk.UserID `json:"userID"` + Type ResourceType `json:"type"` + GPU Range `json:"gpu"` + GPUNumber int `json:"gpuNumber"` + CPU Range `json:"cpu"` + Memory Range `json:"memory"` + Storage Range `json:"storage"` + Ids []string `json:"ids"` +} + +type JobInfo struct { + TaskID string `json:"taskID"` + JobSubmitInfo JobSubmitInfo `json:"jobSubmitInfo"` + ResultFiles []ResultFile `json:"resultFiles"` +} + +type ResultFile struct { + ClusterID schsdk.ClusterID `json:"clusterID"` + Objects []cdssdk.Object `json:"objects"` +} + +type JobSubmitInfo interface { + Noop() +} + +type JobSubmitInfoBase struct { +} + +var JobSubmitInfoTypeUnion = types.NewTypeUnion[JobSubmitInfo]( + (*PCMJobSubmitInfo)(nil), + (*HPCJobSubmitInfo)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobSubmitInfoTypeUnion, "type") + +func (d *JobSubmitInfoBase) Noop() {} + +type PCMJobSubmitInfo struct { + serder.Metadata `union:"pcm"` + JobSubmitInfoBase + Type string `json:"type"` + Info CreateAIJobReq `json:"info"` +} + +type HPCJobSubmitInfo struct { + serder.Metadata `union:"hpc"` + JobSubmitInfoBase + Type string `json:"type"` + Info hpc.CreateHPCJobReq `json:"info"` +} + +type Range struct { + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +type ResourcePriority interface { + Noop() +} + +type ResourcePriorityBase struct { +} + +var ResourcePriorityTypeUnion = types.NewTypeUnion[ResourcePriority]( + (*RegionPriority)(nil), + (*ChipPriority)(nil), + (*BiasPriority)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&ResourcePriorityTypeUnion, "type") + +func (d *ResourcePriorityBase) Noop() {} + +type RegionPriority struct { + serder.Metadata `union:"region"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type ChipPriority struct { + serder.Metadata `union:"chip"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type BiasPriority struct { + serder.Metadata `union:"bias"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type TaskMessage struct { + Status string `json:"status"` + Message string `json:"message"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` +} + +type UploadParams struct { + DataType string `json:"dataType"` + UploadInfo UploadInfo `json:"uploadInfo"` +} + +type UploadInfo interface { + Noop() +} + +var UploadInfoTypeUnion = types.NewTypeUnion[UploadInfo]( + (*LocalUploadInfo)(nil), + (*RemoteUploadInfo)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type") + +type LocalUploadInfo struct { + serder.Metadata `union:"local"` + UploadInfoBase + Type string `json:"type"` + LocalPath string `json:"localPath"` + ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` +} + +type RemoteUploadInfo struct { + serder.Metadata `union:"url"` + UploadInfoBase + Type string `json:"type"` + Url string `json:"url"` + Branch string `json:"branch"` + DataName string `json:"dataName"` + Cluster schsdk.ClusterID `json:"clusterID"` +} + +type UploadInfoBase struct{} + +func (d *UploadInfoBase) Noop() {} + +type UploadPriority interface { + Noop() +} + +var UploadPriorityTypeUnion = types.NewTypeUnion[UploadPriority]( + (*Preferences)(nil), + (*SpecifyCluster)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadPriorityTypeUnion, "type") + +type Preferences struct { + serder.Metadata `union:"preference"` + UploadPriorityBase + Type string `json:"type"` + ResourcePriorities []ResourcePriority `json:"priorities"` +} + +type SpecifyCluster struct { + serder.Metadata `union:"specify"` + UploadPriorityBase + Type string `json:"type"` + Clusters []schsdk.ClusterID `json:"clusters"` +} + +type UploadPriorityBase struct{} + +func (d *UploadPriorityBase) Noop() {} + +type QueryData struct { + DataType string `json:"dataType" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Path string `json:"path"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` + OrderBy string `json:"orderBy" binding:"required"` + PackageName string `json:"packageName"` +} + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*DatasetBinding)(nil), + (*ModelBinding)(nil), + (*CodeBinding)(nil), + (*ImageBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct { + RootPath string `json:"rootPath"` +} + +func (d *DataBindingBase) Noop() {} + +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + OperateType string `json:"operateType"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints int64 `json:"points"` +} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + OperateType string `json:"operateType"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` +} + +type CodeBinding struct { + serder.Metadata `union:"code"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + OperateType string `json:"operateType"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Description string `json:"description"` + ImageID schsdk.ImageID `json:"imageID"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` + // 当集群为openi的时候，需要传入分支 + Branch string `json:"branch"` +} + +//type ImageBinding struct { +// serder.Metadata `union:"image"` +// DataBindingBase +// Type string `json:"type"` +// Name string `json:"name"` +// ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` +// Description string `json:"description"` +// Architecture string `json:"architecture"` +// ResourceType string `json:"resourceType"` +// Tags []string `json:"tags"` +// PackageID cdssdk.PackageID `json:"packageID"` +//} + +type ImageBinding struct { + serder.Metadata `union:"image"` + DataBindingBase + Type string `json:"type"` + ID int64 `json:"id"` + OperateType string `json:"operateType"` + Name string `json:"name"` + IDType string `json:"idType"` + ImageID string `json:"imageID"` + ClusterID schsdk.ClusterID `json:"clusterID"` +} + +type Image struct { + ImageID schsdk.ImageID `json:"imageID" gorm:"column:ImageID;primaryKey"` + Name string `json:"name" gorm:"column:Name"` + CreateTime time.Time `json:"createTime" gorm:"column:CreateTime"` + ClusterImage []ClusterImage `gorm:"foreignKey:image_id;references:ImageID" json:"clusterImages"` +} + +type ClusterImage struct { + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + OriginImageType string `gorm:"column:origin_image_type" json:"originImageType"` + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + OriginImageName string `gorm:"column:origin_image_name" json:"originImageName"` + ClusterImageCard []ClusterImageCard `gorm:"foreignKey:origin_image_id;references:origin_image_id" json:"cards"` +} + +func (ClusterImage) TableName() string { + return "cluster_image" +} + +type ClusterImageCard struct { + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + Card string `gorm:"column:card" json:"card"` +} + +func (ClusterImageCard) TableName() string { + return "cluster_image_card" +} + +type QueryBindingFilters struct { + Status string `json:"status"` + Name string `json:"name"` +} + +type QueryBindingDataParam interface { + Noop() +} + +var QueryBindingDataParamTypeUnion = types.NewTypeUnion[QueryBindingDataParam]( + (*PrivateLevel)(nil), + (*ApplyLevel)(nil), + (*PublicLevel)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&QueryBindingDataParamTypeUnion, "type") + +type QueryBindingDataParamBase struct{} + +func (d *QueryBindingDataParamBase) Noop() {} + +type PrivateLevel struct { + serder.Metadata `union:"private"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + BindingID int64 `json:"bindingID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type ApplyLevel struct { + serder.Metadata `union:"apply"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type PublicLevel struct { + serder.Metadata `union:"public"` + QueryBindingDataParamBase + UserID cdssdk.UserID `json:"userID" binding:"required"` + Type string `json:"type" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go new file mode 100644 index 0000000..8d78ddf --- /dev/null +++ b/sdks/scheduler/jobflow.go @@ -0,0 +1,102 @@ +package schsdk + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) + +type FlowData struct { + Nodes []Node `json:"nodes"` + Edges []Edge `json:"edges"` +} + +type Node struct { + ID string `json:"id"` + Type string `json:"type"` + X float64 `json:"x"` + Y float64 `json:"y"` + Properties JobInfo `json:"properties"` + Text *Text `json:"text,omitempty"` // 有些节点没有 text 字段 +} + +type Edge struct { + ID string `json:"id"` + Type string `json:"type"` + Properties interface{} `json:"properties"` // 为空对象 {} + SourceNodeID string `json:"sourceNodeId"` + TargetNodeID string `json:"targetNodeId"` + SourceAnchorID string `json:"sourceAnchorId"` + TargetAnchorID string `json:"targetAnchorId"` + StartPoint Point `json:"startPoint"` + EndPoint Point `json:"endPoint"` + PointsList []Point `json:"pointsList"` + Text *Text `json:"text,omitempty"` // 有些 edge 有文字标签 +} + +type Point struct { + X float64 `json:"x"` + Y float64 `json:"y"` +} + +type Text struct { + X float64 `json:"x"` + Y float64 `json:"y"` + Value string `json:"value"` +} + +type JobFlowDAO struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"id"` + UserID cdssdk.UserID `gorm:"column:user_id;not null" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content string `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('pending','running','failed','success');default:'pending'" json:"status"` + JobType string `gorm:"column:job_type;size:255" json:"jobType"` + UpdatedAt time.Time `gorm:"column:update_at;autoUpdateTime" json:"updatedAt"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` +} + +type JobFlow struct { + ID int64 `json:"id"` + UserID cdssdk.UserID `json:"userID"` + Name string `json:"name"` + Description string `json:"description"` + Content FlowData `json:"content"` + Status string `json:"status"` + JobType string `json:"jobType"` + UpdatedAt time.Time `json:"updatedAt"` + CreatedAt time.Time `json:"createdAt"` +} + +type JobFlowRunStatus struct { + RunID JobSetID `gorm:"column:run_id;primaryKey" json:"runID"` + NodeType string `gorm:"column:node_type;size:100" json:"nodeType"` + NodeID string `gorm:"column:node_id;size:255" json:"nodeID"` + Status string `gorm:"column:status;type:enum('pending','running','fail','success')" json:"status"` + RunOutput string `gorm:"column:run_output;type:text" json:"runOutput"` + RunLog string `gorm:"column:run_log;type:text" json:"runLog"` +} + +type JobFlowRunDAO struct { + ID JobSetID `gorm:"column:id;primaryKey;" json:"runID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content string `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + Token string `gorm:"column:token;size:255" json:"token"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` + FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` +} + +type JobFlowRun struct { + ID JobSetID `gorm:"column:id;primaryKey;" json:"runID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content FlowData `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + Token string `gorm:"column:token;size:255" json:"token"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` + FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` +} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index b6821e7..8262d75 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -8,25 +8,36 @@ import ( const ( JobTypeNormal = "Normal" + JobTypePCM = "PCM" JobTypeResource = "Resource" JobTypeInstance = "Instance" JobTypeFinetuning = "Finetuning" JobTypeDataPreprocess = "DataPreprocess" + JobTypeDataReturn = "DataReturn" FileInfoTypePackage = "Package" FileInfoTypeLocalFile = "LocalFile" FileInfoTypeResource = "Resource" FileInfoTypeImage = "Image" + FILE = "file" + FOLDER = "folder" + MemoryUtilization = "MemoryUtilization" GPUUtilization = "GPUUtilization" CPUUtilization = "CPUUtilization" + + MethodPost = "POST" + MethodGet = "GET" + CodeSuccess = 200 ) type JobID string type JobSetID string +type DataID int64 + type ImageID int64 // 计算中心ID @@ -38,12 +49,17 @@ type ECSInstanceID string type NodeID int64 type Address string +type ClusterID string + type JobSetInfo struct { Jobs []JobInfo `json:"jobs"` } type JobInfo interface { GetLocalJobID() string + GetTargetLocalJobIDs() []string + SetTargetLocalJob(info TargetJobInfo) + GetTargetInputParams(targetID string) map[string]string } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -54,17 +70,59 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*UpdateMultiInstanceJobInfo)(nil), (*FinetuningJobInfo)(nil), (*DataPreprocessJobInfo)(nil), + (*AIJobInfo)(nil), + (*HPCJobInfo)(nil), + (*BindingJobInfo)(nil), + (*PCMInferenceJobInfo)(nil), + (*CompleteJobInfo)(nil), + (*StartJobInfo)(nil), + (*NotifyJobInfo)(nil), + (*StopInferenceJobInfo)(nil), + (*UploadJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` + LocalJobID string `json:"localJobID"` + TargetJob []TargetJobInfo `json:"targetJob"` +} + +type TargetJobInfo struct { + TargetJobID string `json:"targetJobID"` + InputParams map[string]string `json:"inputParams"` } func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } +func (i *JobInfoBase) GetTargetInputParams(targetID string) map[string]string { + for _, v := range i.TargetJob { + if v.TargetJobID == targetID { + return v.InputParams + } + } + return nil +} + +func (i *JobInfoBase) GetTargetLocalJobIDs() []string { + var IDs []string + for _, v := range i.TargetJob { + IDs = append(IDs, v.TargetJobID) + } + return IDs +} + +func (i *JobInfoBase) SetTargetLocalJob(info TargetJobInfo) { + for _, target := range i.TargetJob { + // 已经存在，则不用再添加 + if target.TargetJobID == info.TargetJobID { + return + } + } + i.TargetJob = append(i.TargetJob, info) +} + type NormalJobInfo struct { serder.Metadata `union:"Normal"` JobInfoBase @@ -76,6 +134,301 @@ type NormalJobInfo struct { ModelJobInfo ModelJobInfo `json:"modelJobInfo"` } +type PCMInferenceJobInfo struct { + serder.Metadata `union:"PCM_Inference"` + JobInfoBase + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` + BindingID DataID `json:"bindingID"` + ResourceChoice ResourceChoice `json:"resourceChoice"` +} + +type StopInferenceJobInfo struct { + serder.Metadata `union:"StopInference"` + JobInfoBase + Type string `json:"type"` + Url string `json:"url"` +} + +type AIJobInfo struct { + serder.Metadata `union:"AI"` + JobInfoBase + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` + ResourceChoice ResourceChoice `json:"resourceChoice"` +} + +type CompleteJobInfo struct { + serder.Metadata `union:"Complete"` + JobInfoBase + Type string `json:"type"` +} + +type StartJobInfo struct { + serder.Metadata `union:"Start"` + JobInfoBase + Type string `json:"type"` +} + +type UploadJobInfo struct { + serder.Metadata `union:"Upload"` + JobInfoBase + Type string `json:"type"` + DataType string `json:"dataType"` +} + +type NotifyJobInfo struct { + serder.Metadata `union:"Notify"` + JobInfoBase + Type string `json:"type"` + RequestType string `json:"requestType"` + Url string `json:"url"` + Body any `json:"body"` + Headers map[string]string `json:"headers"` +} + +type ResourceChoice struct { + Type string `json:"type"` + ResourceScopes []ResourceScope `json:"resourceScopes"` +} + +type ResourceScope struct { + Name string `json:"name"` + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +type BindingJobInfo struct { + serder.Metadata `union:"Binding"` + JobInfoBase + Type string `json:"type"` + Info DataBinding `json:"info"` + Name string `json:"name"` // 临时使用 +} + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*ModelBinding)(nil), + (*DatasetBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct{} + +func (d *DataBindingBase) Noop() {} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + ClusterIDs []ClusterID `json:"clusterIDs"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + RepositoryName string `json:"repositoryName"` +} + +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + OperateType string `json:"operateType"` + ClusterIDs []ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints int64 `json:"points"` +} + +type HPCJobInfo struct { + serder.Metadata `union:"HPC"` + JobInfoBase + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + ClusterID ClusterID `json:"clusterID"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + ScriptContent string `json:"scriptContent"` + Parameters HPCParameter `json:"parameters"` +} + +type HPCParameter struct { + JobName string `json:"jobName"` + JobDir string `json:"jobDir"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + BamFile string `json:"bamFile"` + HashType string `json:"hashType"` + AttackMode string `json:"attackMode"` + HashInput string `json:"hashInput"` + Mask string `json:"mask"` + Dictionary string `json:"dictionary"` + Dictionary2 string `json:"dictionary2"` + HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` +} + +type HPCBindingFile struct { + ParamName string `json:"paramName"` + Resource HPCFile `json:"resource"` +} + +type HPCFile interface { + Noop() +} + +var HPCFileTypeUnion = types.NewTypeUnion[HPCFile]( + (*HPCObject)(nil), + (*HPCPath)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&HPCFileTypeUnion, "type") + +type HPCFileBase struct{} + +func (d *HPCFileBase) Noop() {} + +type HPCObject struct { + serder.Metadata `union:"object"` + HPCFileBase + Type string `json:"type"` + ObjectID cdssdk.ObjectID `json:"objectID"` +} + +type HPCPath struct { + serder.Metadata `union:"path"` + HPCFileBase + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` + Path string `json:"path"` +} + +type JobResources struct { + //任务分配策略：负载均衡、积分优先、随机分配等，dataLocality, leastLoadFirst + ScheduleStrategy string `json:"scheduleStrategy"` + Clusters []ClusterInfo `json:"clusters"` +} + +type ClusterInfo struct { + ClusterID ClusterID `json:"clusterID"` + Resources []JobResource `json:"resources"` + //Files JobFilesInfo `json:"files"` + Code JobFileInfo `json:"code"` + Runtime PCMJobRuntimeInfo `json:"runtime"` +} + +type PCMJobRuntimeInfo struct { + Command string `json:"command"` + Envs map[string]interface{} `json:"envs"` + Params map[string]interface{} `json:"params"` +} + +//type Resource struct { +// Resource []JobResource `json:"resource"` +//} + +type JobResource interface { + Noop() +} + +var JobResourceTypeUnion = types.NewTypeUnion[JobResource]( + (*CPU)(nil), + (*GPU)(nil), + (*NPU)(nil), + (*MLU)(nil), + (*DCU)(nil), + (*MEMORY)(nil), + (*PRICE)(nil), + (*STORAGE)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type") + +type JobResourceBase struct{} + +func (d *JobResourceBase) Noop() {} + +type CPU struct { + serder.Metadata `union:"CPU"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + +type STORAGE struct { + serder.Metadata `union:"STORAGE"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + +type GPU struct { + serder.Metadata `union:"GPU"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + +type NPU struct { + serder.Metadata `union:"NPU"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + +type MEMORY struct { + serder.Metadata `union:"MEMORY"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + +type DCU struct { + serder.Metadata `union:"DCU"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + +type MLU struct { + serder.Metadata `union:"MLU"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + +type PRICE struct { + serder.Metadata `union:"PRICE"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + // FinetuningJobInfo 模型微调 type FinetuningJobInfo struct { serder.Metadata `union:"Finetuning"` @@ -102,9 +455,10 @@ type DataPreprocessJobInfo struct { type DataReturnJobInfo struct { serder.Metadata `union:"DataReturn"` JobInfoBase - Type string `json:"type"` - BucketID cdssdk.BucketID `json:"bucketID"` - TargetLocalJobID string `json:"targetLocalJobID"` + Type string `json:"type"` + BucketID cdssdk.BucketID `json:"bucketID"` + TargetLocalJobID string `json:"targetLocalJobID"` + ReportMessage TrainJobStatusReport `json:"report"` } // MultiInstanceJobInfo 多实例(推理任务) @@ -154,6 +508,7 @@ type JobFilesInfo struct { Dataset JobFileInfo `json:"dataset"` Code JobFileInfo `json:"code"` Image JobFileInfo `json:"image"` + Model JobFileInfo `json:"model"` } type JobFileInfo interface { @@ -165,6 +520,7 @@ var FileInfoTypeUnion = types.NewTypeUnion[JobFileInfo]( (*LocalJobFileInfo)(nil), (*DataReturnJobFileInfo)(nil), (*ImageJobFileInfo)(nil), + (*BindingJobFileInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&FileInfoTypeUnion, "type") @@ -172,6 +528,15 @@ type JobFileInfoBase struct{} func (i *JobFileInfoBase) Noop() {} +type BindingJobFileInfo struct { + serder.Metadata `union:"Binding"` + JobFileInfoBase + Type string `json:"type"` + BindingID int64 `json:"bindingID"` + // 用于参数回显 + BindingName string `json:"bindingName"` +} + type PackageJobFileInfo struct { serder.Metadata `union:"Package"` JobFileInfoBase @@ -198,11 +563,14 @@ type ImageJobFileInfo struct { JobFileInfoBase Type string `json:"type"` ImageID ImageID `json:"imageID"` + // 用于参数回显 + ImageName string `json:"imageName"` } type JobRuntimeInfo struct { Command string `json:"command"` Envs []KVPair `json:"envs"` + Params []KVPair `json:"params"` } type KVPair struct { @@ -305,3 +673,99 @@ type InferencePlatform struct { SimilarityThreshold string `json:"similarityThreshold"` EntriesPerFile string `json:"entriesPerFile"` } + +type JobOutput interface { + Output2() +} + +var JobOutputTypeUnion = types.NewTypeUnion[JobOutput]( + (*AIJobOutput)(nil), + (*BindingJobOutput)(nil), + (*UploadJobOutput)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobOutputTypeUnion, "type") + +type JobOutputBase struct{} + +func (d *JobOutputBase) Output2() {} + +type PublicOutput struct { + serder.Metadata `union:"object"` + JobOutputBase + Type string `json:"type"` +} + +type HPCOutput struct { + serder.Metadata `union:"HPCSlurm"` + JobOutputBase + Type string `json:"type"` + Output string `json:"output"` +} + +type AIJobOutput struct { + serder.Metadata `union:"AI"` + JobOutputBase + Type string `json:"type"` + Output string `json:"output"` +} + +type BindingJobOutput struct { + serder.Metadata `union:"binding"` + JobOutputBase + Type string `json:"type"` + BindingID DataID `json:"bindingID"` +} + +type UploadJobOutput struct { + serder.Metadata `union:"upload"` + JobOutputBase + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` +} + +type DataReturnJobOutput struct { + serder.Metadata `union:"DataReturn"` + JobOutputBase + Type string `json:"type"` + ReportMessage TrainJobStatusReport `json:"report"` + PackageID cdssdk.PackageID `json:"packageID"` +} + +type JobStatusReport interface { + Report() +} + +var JobStatusReportTypeUnion = types.NewTypeUnion[JobStatusReport]( + (*TrainJobStatusReport)(nil), + (*InferenceJobStatusReport)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobStatusReportTypeUnion, "type") + +type JobStatusReportBase struct{} + +func (d *JobStatusReportBase) Report() {} + +type TrainJobStatusReport struct { + serder.Metadata `union:"Train"` + JobStatusReportBase + Type string `json:"type"` + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID ClusterID `json:"clusterID"` + Output string `json:"output"` +} + +type InferenceJobStatusReport struct { + serder.Metadata `union:"Inference"` + JobStatusReportBase + Type string `json:"type"` + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + URL string `json:"url"` +} diff --git a/sdks/sdks.go b/sdks/sdks.go index c556cd5..6f303c9 100644 --- a/sdks/sdks.go +++ b/sdks/sdks.go @@ -1,6 +1,19 @@ package sdks -import "fmt" +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/google/go-querystring/query" + "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/io2" + "gitlink.org.cn/cloudream/common/utils/serder" +) type CodeMessageError struct { Code string @@ -10,3 +23,179 @@ type CodeMessageError struct { func (e *CodeMessageError) Error() string { return fmt.Sprintf("code: %s, message: %s", e.Code, e.Message) } + +type RequestParam struct { + Method string + Path string + Query url.Values + Header http.Header + Body RequestBody +} + +func (p *RequestParam) MakeRequest(baseURL string) (*http.Request, error) { + var body io.ReadCloser + bodyLen := int64(0) + if p.Body != nil { + body = p.Body.IntoStream() + bodyLen = p.Body.Length() + } + + req, err := http.NewRequest(p.Method, joinUrlUnsafe(baseURL, p.Path), body) + if err != nil { + return nil, err + } + req.ContentLength = bodyLen + req.URL.RawQuery = p.Query.Encode() + + if p.Header != nil { + req.Header = p.Header + } + + return req, nil +} + +func (p *RequestParam) Do(baseURL string, cli *http.Client) (*http.Response, error) { + req, err := p.MakeRequest(baseURL) + if err != nil { + return nil, err + } + + return cli.Do(req) +} + +func joinUrlUnsafe(base string, path string) string { + if strings.HasSuffix(base, "/") { + if strings.HasPrefix(path, "/") { + return base + path[1:] + } + + return base + path + } + + if strings.HasPrefix(path, "/") { + return base + path + } + + return base + "/" + path +} + +type RequestBody interface { + // 请求体长度，如果长度未知，返回-1 + Length() int64 + // 将内部值变成一个流，用于发送请求 + IntoStream() io.ReadCloser +} + +type StringBody struct { + Value string +} + +func (s *StringBody) Length() int64 { + return int64(len(s.Value)) +} + +func (s *StringBody) IntoStream() io.ReadCloser { + return io.NopCloser(bytes.NewReader([]byte(s.Value))) +} + +type BytesBody struct { + Value []byte +} + +func (b *BytesBody) Length() int64 { + return int64(len(b.Value)) +} + +func (b *BytesBody) IntoStream() io.ReadCloser { + return io.NopCloser(bytes.NewReader(b.Value)) +} + +type StreamBody struct { + Stream io.ReadCloser + LengthHint int64 // 长度提示，如果长度未知，可以设置为-1 +} + +func (s *StreamBody) Length() int64 { + return s.LengthHint +} + +func (s *StreamBody) IntoStream() io.ReadCloser { + return s.Stream +} + +type APIRequest interface { + MakeParam() *RequestParam +} + +func MakeJSONParam(method string, path string, body any) *RequestParam { + data, err := serder.ObjectToJSONEx(body) + if err != nil { + // 开发人员应该保证param是可序列化的 + panic(err) + } + + return &RequestParam{ + Method: method, + Path: path, + Body: &BytesBody{Value: data}, + } +} + +func MakeQueryParam(method string, path string, q any) *RequestParam { + values, err := query.Values(q) + if err != nil { + // 开发人员应该保证param是可序列化的 + panic(err) + } + + return &RequestParam{ + Method: method, + Path: path, + Query: values, + } +} + +type APIResponse interface { + ParseResponse(resp *http.Response) error +} + +type CodeDataResponse[T any] struct { + Code string `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +func (r *CodeDataResponse[T]) Unwarp() (T, error) { + if r.Code == errorcode.OK { + return r.Data, nil + } + var def T + return def, &CodeMessageError{Code: r.Code, Message: r.Message} +} + +func ParseCodeDataJSONResponse[T any](resp *http.Response, ret T) error { + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var err error + var r CodeDataResponse[T] + r.Data = ret + + if err = serder.JSONToObjectStreamExRaw(resp.Body, &r); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if r.Code != errorcode.OK { + return &CodeMessageError{Code: r.Code, Message: r.Message} + } + + return nil + } + + cont, err := io2.ReadMost(resp.Body, 200) + if err != nil { + return fmt.Errorf("unknow response content type: %s, status: %d", contType, resp.StatusCode) + } + strCont := string(cont) + + return fmt.Errorf("unknow response content type: %s, status: %d, body[:200]: %s", contType, resp.StatusCode, strCont) +} diff --git a/sdks/storage/cdsapi/bucket.go b/sdks/storage/cdsapi/bucket.go index 2cb2e9c..f0c1ad1 100644 --- a/sdks/storage/cdsapi/bucket.go +++ b/sdks/storage/cdsapi/bucket.go @@ -1,11 +1,10 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) type BucketService struct { @@ -19,36 +18,24 @@ func (c *Client) Bucket() *BucketService { const BucketGetByNamePath = "/bucket/getByName" type BucketGetByName struct { - UserID cdssdk.UserID `json:"userID" form:"userID" binding:"required"` - Name string `json:"name" form:"name" binding:"required"` + UserID cdssdk.UserID `url:"userID" form:"userID" binding:"required"` + Name string `url:"name" form:"name" binding:"required"` } + +func (r *BucketGetByName) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, BucketGetByNamePath, r) +} + type BucketGetByNameResp struct { Bucket cdssdk.Bucket `json:"bucket"` } -func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) { - url, err := url.JoinPath(c.baseURL, BucketGetByNamePath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketGetByNameResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketGetByNameResp{}) } const BucketCreatePath = "/bucket/create" @@ -58,33 +45,20 @@ type BucketCreate struct { Name string `json:"name" binding:"required"` } +func (r *BucketCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, BucketCreatePath, r) +} + type BucketCreateResp struct { Bucket cdssdk.Bucket `json:"bucket"` } -func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) { - url, err := url.JoinPath(c.baseURL, BucketCreatePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketCreateResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *BucketCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketCreateResp{}) } const BucketDeletePath = "/bucket/delete" @@ -94,64 +68,38 @@ type BucketDelete struct { BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` } -type BucketDeleteResp struct{} - -func (c *BucketService) Delete(req BucketDelete) error { - url, err := url.JoinPath(c.baseURL, BucketDeletePath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +func (r *BucketDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, BucketDeletePath, r) +} - codeResp, err := ParseJSONResponse[response[BucketDeleteResp]](resp) - if err != nil { - return err - } +type BucketDeleteResp struct{} - if codeResp.Code == errorcode.OK { - return nil - } +func (r *BucketDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return codeResp.ToError() +func (c *BucketService) Delete(req BucketDelete) error { + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const BucketListUserBucketsPath = "/bucket/listUserBuckets" type BucketListUserBucketsReq struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` +} + +func (r *BucketListUserBucketsReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, BucketListUserBucketsPath, r) } type BucketListUserBucketsResp struct { Buckets []cdssdk.Bucket `json:"buckets"` } +func (r *BucketListUserBucketsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *BucketService) ListUserBuckets(req BucketListUserBucketsReq) (*BucketListUserBucketsResp, error) { - url, err := url.JoinPath(c.baseURL, BucketListUserBucketsPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketListUserBucketsResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketListUserBucketsResp{}) } diff --git a/sdks/storage/cdsapi/cache.go b/sdks/storage/cdsapi/cache.go index 609cdd0..b71725d 100644 --- a/sdks/storage/cdsapi/cache.go +++ b/sdks/storage/cdsapi/cache.go @@ -1,11 +1,10 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) const CacheMovePackagePath = "/cache/movePackage" @@ -15,29 +14,17 @@ type CacheMovePackageReq struct { PackageID cdssdk.PackageID `json:"packageID"` StorageID cdssdk.StorageID `json:"storageID"` } + +func (r *CacheMovePackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, CacheMovePackagePath, r) +} + type CacheMovePackageResp struct{} +func (r *CacheMovePackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) CacheMovePackage(req CacheMovePackageReq) (*CacheMovePackageResp, error) { - url, err := url.JoinPath(c.baseURL, CacheMovePackagePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[CacheMovePackageResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } - - return nil, jsonResp.ToError() + return JSONAPI(c.cfg, http.DefaultClient, &req, &CacheMovePackageResp{}) } diff --git a/sdks/storage/cdsapi/client.go b/sdks/storage/cdsapi/client.go index 40af4dd..8e12be3 100644 --- a/sdks/storage/cdsapi/client.go +++ b/sdks/storage/cdsapi/client.go @@ -18,12 +18,12 @@ func (r *response[T]) ToError() *sdks.CodeMessageError { } type Client struct { - baseURL string + cfg *Config } func NewClient(cfg *Config) *Client { return &Client{ - baseURL: cfg.URL, + cfg: cfg, } } diff --git a/sdks/storage/cdsapi/config.go b/sdks/storage/cdsapi/config.go index 484808c..448039e 100644 --- a/sdks/storage/cdsapi/config.go +++ b/sdks/storage/cdsapi/config.go @@ -1,5 +1,7 @@ package cdsapi type Config struct { - URL string `json:"url"` + URL string `json:"url"` + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` } diff --git a/sdks/storage/cdsapi/hub.go b/sdks/storage/cdsapi/hub.go index 09fa7bd..20a9d18 100644 --- a/sdks/storage/cdsapi/hub.go +++ b/sdks/storage/cdsapi/hub.go @@ -1,44 +1,30 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) var HubGetHubsPath = "/hub/getHubs" type HubGetHubsReq struct { - HubIDs []cdssdk.HubID `json:"hubIDs"` + HubIDs []cdssdk.HubID `form:"hubIDs" url:"hubIDs"` +} + +func (r *HubGetHubsReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, HubGetHubsPath, r) } type HubGetHubsResp struct { - Hubs []cdssdk.Hub `json:"hubs"` + Hubs []*cdssdk.Hub `json:"hubs"` +} + +func (r *HubGetHubsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) } func (c *Client) HubGetHubs(req HubGetHubsReq) (*HubGetHubsResp, error) { - url, err := url.JoinPath(c.baseURL, HubGetHubsPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[HubGetHubsResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } - - return nil, jsonResp.ToError() + return JSONAPI(c.cfg, http.DefaultClient, &req, &HubGetHubsResp{}) } diff --git a/sdks/storage/cdsapi/hub_io.go b/sdks/storage/cdsapi/hub_io.go index bd5f1a3..826c425 100644 --- a/sdks/storage/cdsapi/hub_io.go +++ b/sdks/storage/cdsapi/hub_io.go @@ -24,7 +24,7 @@ type GetStreamReq struct { } func (c *Client) GetStream(req GetStreamReq) (io.ReadCloser, error) { - targetUrl, err := url.JoinPath(c.baseURL, GetStreamPath) + targetUrl, err := url.JoinPath(c.cfg.URL, GetStreamPath) if err != nil { return nil, err } @@ -65,7 +65,7 @@ type SendStreamInfo struct { } func (c *Client) SendStream(req SendStreamReq) error { - targetUrl, err := url.JoinPath(c.baseURL, SendStreamPath) + targetUrl, err := url.JoinPath(c.cfg.URL, SendStreamPath) if err != nil { return err } @@ -133,7 +133,7 @@ type ExecuteIOPlanReq struct { } func (c *Client) ExecuteIOPlan(req ExecuteIOPlanReq) error { - targetUrl, err := url.JoinPath(c.baseURL, ExecuteIOPlanPath) + targetUrl, err := url.JoinPath(c.cfg.URL, ExecuteIOPlanPath) if err != nil { return err } @@ -171,7 +171,7 @@ type SendVarReq struct { } func (c *Client) SendVar(req SendVarReq) error { - targetUrl, err := url.JoinPath(c.baseURL, SendVarPath) + targetUrl, err := url.JoinPath(c.cfg.URL, SendVarPath) if err != nil { return err } @@ -214,7 +214,7 @@ type GetVarResp struct { } func (c *Client) GetVar(req GetVarReq) (*GetVarResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, GetVarPath) + targetUrl, err := url.JoinPath(c.cfg.URL, GetVarPath) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 32d7243..718c3ba 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -1,15 +1,20 @@ package cdsapi import ( + "context" "fmt" "io" "mime" + "net/http" "net/url" "strings" "time" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -25,41 +30,58 @@ func (c *Client) Object() *ObjectService { } } -const ObjectListPath = "/object/list" +const ObjectListPathByPath = "/object/listByPath" -type ObjectList struct { - UserID cdssdk.UserID `form:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" binding:"required"` - Path string `form:"path"` // 允许为空字符串 - IsPrefix bool `form:"isPrefix"` +type ObjectListByPath struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID" json:"userID"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID" json:"packageID"` + Path string `form:"path" url:"path" json:"path"` // 允许为空字符串 + IsPrefix bool `form:"isPrefix" url:"isPrefix" json:"isPrefix"` + NoRecursive bool `form:"noRecursive" url:"noRecursive" json:"noRecursive"` // 仅当isPrefix为true时有效，表示仅查询直接属于Prefix下的对象，对于更深的对象，返回它们的公共前缀 + MaxKeys int `form:"maxKeys" url:"maxKeys" json:"maxKeys"` + ContinuationToken string `form:"continuationToken" url:"continuationToken" json:"continuationToken"` // 用于分页，如果为空字符串，表示从头开始 } -type ObjectListResp struct { - Objects []cdssdk.Object `json:"objects"` + +func (r *ObjectListByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectListPathByPath, r) } -func (c *ObjectService) List(req ObjectList) (*ObjectListResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectListPath) - if err != nil { - return nil, err - } +type ObjectListByPathResp struct { + CommonPrefixes []string `json:"commonPrefixes"` // 仅在IsPrefix为true且NoRecursive为true时有效，包含更深层对象的shared prefix + Objects []cdssdk.Object `json:"objects"` // 如果IsPrefix为true且NoRecursive为false，则返回所有匹配的对象，否则只返回直接属于Prefix下的对象 + IsTruncated bool `json:"isTruncated"` // 是否还有更多对象 + NextContinuationToken string `json:"nextContinuationToken"` // 用于分页，如果IsTruncated为true，则下次请求的ContinuationToken为该值 +} - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } +func (r *ObjectListByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectListResp]](resp) - if err != nil { - return nil, err - } +func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectListByPathResp{}) +} - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +const ObjectListByIDsPath = "/object/listByIDs" + +type ObjectListByIDs struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + ObjectIDs []cdssdk.ObjectID `form:"objectIDs" binding:"required" url:"objectIDs"` +} + +func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) +} + +type ObjectListByIDsResp struct { + Objects []*cdssdk.Object `json:"object"` // 与ObjectIDs一一对应，如果某个ID不存在，则对应位置为nil +} - return nil, jsonResp.ToError() +func (r *ObjectListByIDsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectListByIDsResp{}) } const ObjectUploadPath = "/object/upload" @@ -70,10 +92,11 @@ type ObjectUpload struct { } type ObjectUploadInfo struct { - UserID cdssdk.UserID `json:"userID" binding:"required"` - PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - Affinity cdssdk.StorageID `json:"affinity"` - LoadTo []cdssdk.StorageID `json:"loadTo"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + Affinity cdssdk.StorageID `json:"affinity"` + LoadTo []cdssdk.StorageID `json:"loadTo"` + LoadToPath []string `json:"loadToPath"` } type UploadingObject struct { @@ -88,7 +111,11 @@ type ObjectUploadResp struct { } func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUploadPath) + type uploadInfo struct { + Info string `url:"info"` + } + + url, err := url.JoinPath(c.cfg.URL, ObjectUploadPath) if err != nil { return nil, err } @@ -98,16 +125,15 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { return nil, fmt.Errorf("upload info to json: %w", err) } - resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{ - Form: map[string]string{"info": string(infoJSON)}, - Files: iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { + resp, err := PostMultiPart(c.cfg, url, + uploadInfo{Info: string(infoJSON)}, + iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { return &http2.IterMultiPartFile{ FieldName: "files", FileName: src.Path, File: src.File, }, nil - }), - }) + })) if err != nil { return nil, err } @@ -134,25 +160,42 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { const ObjectDownloadPath = "/object/download" type ObjectDownload struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - ObjectID cdssdk.ObjectID `form:"objectID" json:"objectID" binding:"required"` - Offset int64 `form:"offset" json:"offset,omitempty"` - Length *int64 `form:"length" json:"length,omitempty"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `form:"objectID" url:"objectID" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (r *ObjectDownload) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectDownloadPath, r) } + type DownloadingObject struct { Path string File io.ReadCloser } func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) { - url, err := url.JoinPath(c.baseURL, ObjectDownloadPath) + httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL) if err != nil { return nil, err } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) + if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + resp, err := http.DefaultClient.Do(httpReq) if err != nil { return nil, err } @@ -182,22 +225,38 @@ func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) const ObjectDownloadByPathPath = "/object/downloadByPath" type ObjectDownloadByPath struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` - Path string `form:"path" json:"path" binding:"required"` - Offset int64 `form:"offset" json:"offset,omitempty"` - Length *int64 `form:"length" json:"length,omitempty"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` + Path string `form:"path" url:"path" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (r *ObjectDownloadByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectDownloadByPathPath, r) } func (c *ObjectService) DownloadByPath(req ObjectDownloadByPath) (*DownloadingObject, error) { - url, err := url.JoinPath(c.baseURL, ObjectDownloadByPathPath) + httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL) if err != nil { return nil, err } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) + if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + resp, err := http.DefaultClient.Do(httpReq) if err != nil { return nil, err } @@ -240,33 +299,20 @@ type ObjectUpdateInfo struct { Updatings []UpdatingObject `json:"updatings" binding:"required"` } +func (r *ObjectUpdateInfo) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectUpdateInfoPath, r) +} + type ObjectUpdateInfoResp struct { Successes []cdssdk.ObjectID `json:"successes"` } -func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectUpdateInfoResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectUpdateInfoResp{}) } const ObjectUpdateInfoByPathPath = "/object/updateInfoByPath" @@ -278,31 +324,18 @@ type ObjectUpdateInfoByPath struct { UpdateTime time.Time `json:"updateTime" binding:"required"` } -type ObjectUpdateInfoByPathResp struct{} - -func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoByPathPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } +func (r *ObjectUpdateInfoByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectUpdateInfoByPathPath, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoByPathResp]](resp) - if err != nil { - return nil, err - } +type ObjectUpdateInfoByPathResp struct{} - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectUpdateInfoByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectUpdateInfoByPathResp{}) } const ObjectMovePath = "/object/move" @@ -323,33 +356,20 @@ type ObjectMove struct { Movings []MovingObject `json:"movings" binding:"required"` } +func (r *ObjectMove) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectMovePath, r) +} + type ObjectMoveResp struct { Successes []cdssdk.ObjectID `json:"successes"` } -func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectMovePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectMoveResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectMoveResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectMoveResp{}) } const ObjectDeletePath = "/object/delete" @@ -359,31 +379,18 @@ type ObjectDelete struct { ObjectIDs []cdssdk.ObjectID `json:"objectIDs" binding:"required"` } -type ObjectDeleteResp struct{} - -func (c *ObjectService) Delete(req ObjectDelete) error { - url, err := url.JoinPath(c.baseURL, ObjectDeletePath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +func (r *ObjectDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectDeletePath, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectDeleteResp]](resp) - if err != nil { - return err - } +type ObjectDeleteResp struct{} - if jsonResp.Code == errorcode.OK { - return nil - } +func (r *ObjectDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return jsonResp.ToError() +func (c *ObjectService) Delete(req ObjectDelete) error { + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const ObjectDeleteByPathPath = "/object/deleteByPath" @@ -393,64 +400,172 @@ type ObjectDeleteByPath struct { PackageID cdssdk.PackageID `json:"packageID" binding:"required"` Path string `json:"path" binding:"required"` } + +func (r *ObjectDeleteByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectDeleteByPathPath, r) +} + type ObjectDeleteByPathResp struct{} +func (r *ObjectDeleteByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error { - url, err := url.JoinPath(c.baseURL, ObjectDeleteByPathPath) - if err != nil { - return err - } + return JSONAPINoData(c.cfg, http.DefaultClient, &req) +} - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +const ObjectClonePath = "/object/clone" - jsonResp, err := ParseJSONResponse[response[ObjectDeleteByPathResp]](resp) - if err != nil { - return err - } +type ObjectClone struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + Clonings []CloningObject `json:"clonings" binding:"required"` +} - if jsonResp.Code == errorcode.OK { - return nil - } +func (r *ObjectClone) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectClonePath, r) +} + +type CloningObject struct { + ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"` + NewPath string `json:"newPath" binding:"required"` + NewPackageID cdssdk.PackageID `json:"newPackageID" binding:"required"` +} - return jsonResp.ToError() +type ObjectCloneResp struct { + Objects []*cdssdk.Object `json:"objects"` +} + +func (r *ObjectCloneResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectCloneResp{}) } const ObjectGetPackageObjectsPath = "/object/getPackageObjects" type ObjectGetPackageObjects struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` +} + +func (r *ObjectGetPackageObjects) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectGetPackageObjectsPath, r) } + type ObjectGetPackageObjectsResp struct { Objects []cdssdk.Object `json:"objects"` } +func (r *ObjectGetPackageObjectsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectGetPackageObjectsPath) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectGetPackageObjectsResp{}) +} + +const ObjectNewMultipartUploadPath = "/v1/object/newMultipartUpload" + +type ObjectNewMultipartUpload struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + Path string `json:"path" binding:"required"` +} + +func (r *ObjectNewMultipartUpload) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectNewMultipartUploadPath, r) +} + +type ObjectNewMultipartUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (r *ObjectNewMultipartUploadResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectNewMultipartUploadResp{}) +} + +const ObjectUploadPartPath = "/v1/object/uploadPart" + +type ObjectUploadPart struct { + ObjectUploadPartInfo + File io.ReadCloser `json:"-"` +} + +type ObjectUploadPartInfo struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"` + Index int `json:"index"` +} + +type ObjectUploadPartResp struct{} + +func (c *ObjectService) UploadPart(req ObjectUploadPart) (*ObjectUploadPartResp, error) { + url, err := url.JoinPath(c.cfg.URL, ObjectUploadPartPath) if err != nil { return nil, err } - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) + infoJSON, err := serder.ObjectToJSON(req) if err != nil { - return nil, err + return nil, fmt.Errorf("upload info to json: %w", err) } - jsonResp, err := ParseJSONResponse[response[ObjectGetPackageObjectsResp]](resp) + resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{ + Form: map[string]string{"info": string(infoJSON)}, + Files: iterator.Array(&http2.IterMultiPartFile{ + FieldName: "file", + File: req.File, + }), + }) if err != nil { return nil, err } - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var err error + var codeResp response[ObjectUploadPartResp] + if codeResp, err = serder.JSONToObjectStreamEx[response[ObjectUploadPartResp]](resp.Body); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == errorcode.OK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() } - return nil, jsonResp.ToError() + return nil, fmt.Errorf("unknow response content type: %s", contType) +} + +const ObjectCompleteMultipartUploadPath = "/v1/object/completeMultipartUpload" + +type ObjectCompleteMultipartUpload struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"` + Indexes []int `json:"indexes" binding:"required"` +} + +func (r *ObjectCompleteMultipartUpload) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectCompleteMultipartUploadPath, r) +} + +type ObjectCompleteMultipartUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (r *ObjectCompleteMultipartUploadResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectCompleteMultipartUploadResp{}) } diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index 6be57aa..d406fe4 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -1,12 +1,20 @@ package cdsapi import ( + "context" "fmt" + "io" + "mime" + "net/http" "net/url" "strings" + "time" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -23,72 +31,48 @@ func (c *Client) Package() *PackageService { const PackageGetPath = "/package/get" type PackageGetReq struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` } + +func (r *PackageGetReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetPath, r) +} + type PackageGetResp struct { cdssdk.Package } -func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } +func (r *PackageGetResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - codeResp, err := ParseJSONResponse[response[PackageGetResp]](resp) - if err != nil { - return nil, err - } +func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetResp{}) +} - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +const PackageGetByFullNamePath = "/package/getByFullName" - return nil, codeResp.ToError() +type PackageGetByFullName struct { + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + BucketName string `form:"bucketName" url:"bucketName" binding:"required"` + PackageName string `form:"packageName" url:"packageName" binding:"required"` } -const PackageGetByNamePath = "/package/getByName" - -type PackageGetByName struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - BucketName string `form:"bucketName" json:"bucketName" binding:"required"` - PackageName string `form:"packageName" json:"packageName" binding:"required"` +func (r *PackageGetByFullName) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetByFullNamePath, r) } -type PackageGetByNameResp struct { + +type PackageGetByFullNameResp struct { Package cdssdk.Package `json:"package"` } -func (c *PackageService) GetByName(req PackageGetByName) (*PackageGetByNameResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetByNamePath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetByNameResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetByFullNameResp{}) } const PackageCreatePath = "/package/create" @@ -99,33 +83,20 @@ type PackageCreate struct { Name string `json:"name"` } +func (r *PackageCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageCreatePath, r) +} + type PackageCreateResp struct { Package cdssdk.Package `json:"package"` } -func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) { - url, err := url.JoinPath(s.baseURL, PackageCreatePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageCreateResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) { + return JSONAPI(s.cfg, http.DefaultClient, &req, &PackageCreateResp{}) } const PackageCreateLoadPath = "/package/createLoad" @@ -135,19 +106,19 @@ type PackageCreateLoad struct { Files UploadObjectIterator `json:"-"` } type PackageCreateLoadInfo struct { - UserID cdssdk.UserID `json:"userID" binding:"required"` - BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` - Name string `json:"name" binding:"required"` - LoadTo []cdssdk.StorageID `json:"loadTo" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` + Name string `json:"name" binding:"required"` + LoadTo []cdssdk.StorageID `json:"loadTo"` + LoadToPath []string `json:"loadToPath"` } type PackageCreateLoadResp struct { - Package cdssdk.Package `json:"package"` - Objects []cdssdk.Object `json:"objects"` - LoadedDirs []string `json:"loadedDirs"` + Package cdssdk.Package `json:"package"` + Objects []cdssdk.Object `json:"objects"` } func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadResp, error) { - url, err := url.JoinPath(c.baseURL, PackageCreateLoadPath) + url, err := url.JoinPath(c.cfg.URL, PackageCreateLoadPath) if err != nil { return nil, err } @@ -157,16 +128,15 @@ func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadRe return nil, fmt.Errorf("upload info to json: %w", err) } - resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{ - Form: map[string]string{"info": string(infoJSON)}, - Files: iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { + resp, err := PostMultiPart(c.cfg, url, + map[string]string{"info": string(infoJSON)}, + iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { return &http2.IterMultiPartFile{ FieldName: "files", FileName: src.Path, File: src.File, }, nil - }), - }) + })) if err != nil { return nil, err } @@ -183,24 +153,45 @@ func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadRe return nil, codeResp.ToError() } -const PackageDeletePath = "/package/delete" +const PackageDownloadPath = "/v1/package/download" -type PackageDelete struct { - UserID cdssdk.UserID `json:"userID" binding:"required"` - PackageID cdssdk.PackageID `json:"packageID" binding:"required"` +type PackageDownload struct { + UserID cdssdk.UserID `url:"userID" form:"userID" binding:"required"` + PackageID cdssdk.PackageID `url:"packageID" form:"packageID" binding:"required"` } -func (c *PackageService) Delete(req PackageDelete) error { - url, err := url.JoinPath(c.baseURL, PackageDeletePath) +func (r *PackageDownload) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodGet, PackageDownloadPath, r) +} + +type DownloadingPackage struct { + Name string + File io.ReadCloser +} + +func (c *PackageService) Download(req PackageDownload) (*DownloadingPackage, error) { + httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL) if err != nil { - return err + return nil, err } - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) + if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + resp, err := http.DefaultClient.Do(httpReq) if err != nil { - return err + return nil, err } contType := resp.Header.Get("Content-Type") @@ -208,121 +199,134 @@ func (c *PackageService) Delete(req PackageDelete) error { if strings.Contains(contType, http2.ContentTypeJSON) { var codeResp response[any] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return fmt.Errorf("parsing response: %w", err) + return nil, fmt.Errorf("parsing response: %w", err) } - if codeResp.Code == errorcode.OK { - return nil - } + return nil, codeResp.ToError() + } - return codeResp.ToError() + _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) + if err != nil { + return nil, fmt.Errorf("parsing content disposition: %w", err) } - return fmt.Errorf("unknow response content type: %s", contType) + return &DownloadingPackage{ + Name: params["filename"], + File: resp.Body, + }, nil +} + +const PackageDeletePath = "/package/delete" + +type PackageDelete struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` +} + +func (r *PackageDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageDeletePath, r) +} + +type PackageDeleteResp struct{} + +func (r *PackageDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *PackageService) Delete(req PackageDelete) error { + return JSONAPINoData(c.cfg, http.DefaultClient, &req) +} + +const PackageClonePath = "/package/clone" + +type PackageClone struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` + Name string `json:"name" binding:"required"` +} + +func (r *PackageClone) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageClonePath, r) +} + +type PackageCloneResp struct { + Package cdssdk.Package `json:"package"` +} + +func (r *PackageCloneResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageCloneResp{}) } const PackageListBucketPackagesPath = "/package/listBucketPackages" type PackageListBucketPackages struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - BucketID cdssdk.BucketID `form:"bucketID" json:"bucketID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + BucketID cdssdk.BucketID `form:"bucketID" url:"bucketID" binding:"required"` +} + +func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageListBucketPackagesPath, r) } type PackageListBucketPackagesResp struct { Packages []cdssdk.Package `json:"packages"` } -func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { - url, err := url.JoinPath(c.baseURL, PackageListBucketPackagesPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageListBucketPackagesResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageListBucketPackagesResp{}) } const PackageGetCachedStoragesPath = "/package/getCachedStorages" type PackageGetCachedStoragesReq struct { - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` +} + +func (r *PackageGetCachedStoragesReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetCachedStoragesPath, r) } type PackageGetCachedStoragesResp struct { cdssdk.PackageCachingInfo } -func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetCachedStoragesPath) - if err != nil { - return nil, err - } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetCachedStoragesResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageGetCachedStoragesResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetCachedStoragesResp{}) } -const PackageGetLoadedStoragesPath = "/package/getLoadedStorages" +const PackageCalcHashPath = "/v1/package/calcHash" -type PackageGetLoadedStoragesReq struct { - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` +type PackageCalcHash struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` } -type PackageGetLoadedStoragesResp struct { - StorageIDs []cdssdk.StorageID `json:"storageIDs"` +func (r *PackageCalcHash) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageCalcHashPath, r) } -func (c *PackageService) GetLoadedStorages(req PackageGetLoadedStoragesReq) (*PackageGetLoadedStoragesResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetLoadedStoragesPath) - if err != nil { - return nil, err - } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetLoadedStoragesResp]](resp) - if err != nil { - return nil, err - } +type PackageCalcHashResp struct { + cdssdk.PackageHash +} - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageCalcHashResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) CalcHash(req PackageCalcHash) (*PackageCalcHashResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageCalcHashResp{}) } diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go new file mode 100644 index 0000000..99bf8b2 --- /dev/null +++ b/sdks/storage/cdsapi/presigned.go @@ -0,0 +1,157 @@ +package cdsapi + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/google/go-querystring/query" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" +) + +type PresignedService struct { + *Client +} + +func (c *Client) Presigned() *PresignedService { + return &PresignedService{ + Client: c, + } +} + +const PresignedObjectListByPathPath = "/v1/presigned/object/listByPath" + +type PresignedObjectListByPath struct { + ObjectListByPath +} + +func (c *PresignedService) ObjectListByPath(req PresignedObjectListByPath, expireIn int) (string, error) { + return c.presign(req, PresignedObjectListByPathPath, http.MethodGet, expireIn) +} + +const PresignedObjectDownloadByPathPath = "/v1/presigned/object/downloadByPath" + +type PresignedObjectDownloadByPath struct { + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` + Path string `form:"path" url:"path" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) { + return c.presign(req, PresignedObjectDownloadByPathPath, http.MethodGet, expireIn) +} + +const PresignedObjectDownloadPath = "/v1/presigned/object/download" + +type PresignedObjectDownload struct { + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `form:"objectID" url:"objectID" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectDownloadPath, http.MethodGet, expireIn) +} + +const PresignedObjectUploadPath = "/v1/presigned/object/upload" + +type PresignedObjectUpload struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID"` + Path string `form:"path" binding:"required" url:"path"` + Affinity cdssdk.StorageID `form:"affinity" url:"affinity,omitempty"` + LoadTo []cdssdk.StorageID `form:"loadTo" url:"loadTo,omitempty"` + LoadToPath []string `form:"loadToPath" url:"loadToPath,omitempty"` +} + +type PresignedObjectUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectUploadPath, http.MethodPost, expireIn) +} + +const PresignedObjectNewMultipartUploadPath = "/v1/presigned/object/newMultipartUpload" + +type PresignedObjectNewMultipartUpload struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID"` + Path string `form:"path" binding:"required" url:"path"` +} + +type PresignedObjectNewMultipartUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultipartUpload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectNewMultipartUploadPath, http.MethodPost, expireIn) +} + +const PresignedObjectUploadPartPath = "/v1/presigned/object/uploadPart" + +type PresignedObjectUploadPart struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + ObjectID cdssdk.ObjectID `form:"objectID" binding:"required" url:"objectID"` + Index int `form:"index" binding:"required" url:"index"` +} + +type PresignedUploadPartResp struct{} + +func (c *PresignedService) ObjectUploadPart(req PresignedObjectUploadPart, expireIn int) (string, error) { + return c.presign(req, PresignedObjectUploadPartPath, http.MethodPost, expireIn) +} + +const PresignedObjectCompleteMultipartUploadPath = "/v1/presigned/object/completeMultipartUpload" + +type PresignedObjectCompleteMultipartUpload struct { + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + ObjectID cdssdk.ObjectID `form:"objectID" binding:"required" url:"objectID"` + Indexes []int `form:"indexes" binding:"required" url:"indexes"` +} + +type PresignedObjectCompleteMultipartUploadResp struct { + Object cdssdk.Object `json:"object"` +} + +func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectCompleteMultipartUpload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectCompleteMultipartUploadPath, http.MethodPost, expireIn) +} + +func (c *PresignedService) presign(req any, path string, method string, expireIn int) (string, error) { + u, err := url.Parse(c.cfg.URL) + if err != nil { + return "", err + } + u = u.JoinPath(path) + + us, err := query.Values(req) + if err != nil { + return "", err + } + us.Add("X-Expires", fmt.Sprintf("%v", expireIn)) + + u.RawQuery = us.Encode() + + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return "", err + } + + r, err := http.NewRequest(method, u.String(), nil) + if err != nil { + return "", err + } + + signer := v4.NewSigner() + signedURL, _, err := signer.PresignHTTP(context.Background(), cred, r, "", AuthService, AuthRegion, time.Now()) + return signedURL, err +} diff --git a/sdks/storage/cdsapi/presigned_test.go b/sdks/storage/cdsapi/presigned_test.go new file mode 100644 index 0000000..7379951 --- /dev/null +++ b/sdks/storage/cdsapi/presigned_test.go @@ -0,0 +1,178 @@ +package cdsapi + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + "gitlink.org.cn/cloudream/common/pkgs/types" +) + +func Test_Presigned(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectDownloadByPath(PresignedObjectDownloadByPath{ + UserID: 1, + PackageID: 3, + Path: "example.java", + Offset: 1, + Length: types.Ref(int64(100)), + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) + + Convey("上传文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectUpload(PresignedObjectUpload{ + UserID: 1, + PackageID: 3, + Path: "example.java", + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedObjectListByPath(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectListByPath(PresignedObjectListByPath{ + ObjectListByPath: ObjectListByPath{ + UserID: 1, + PackageID: 12, + Path: "a/", + IsPrefix: true, + NoRecursive: true, + MaxKeys: 10, + ContinuationToken: "123456", + }, + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedObjectDownloadByPath(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectDownloadByPath(PresignedObjectDownloadByPath{ + UserID: 1, + PackageID: 3, + Path: "example.java", + // Offset: 1, + // Length: types.Ref(int64(100)), + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedObjectDownload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectDownload(PresignedObjectDownload{ + UserID: 1, + ObjectID: 1039, + // Offset: 1, + // Length: types.Ref(int64(100)), + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedObjectUpload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + }) + + Convey("上传文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectUpload(PresignedObjectUpload{ + UserID: 1, + PackageID: 3, + Path: "example.java", + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedNewMultipartUpload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + }) + + Convey("启动分片上传", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectNewMultipartUpload(PresignedObjectNewMultipartUpload{ + UserID: 1, + PackageID: 3, + Path: "example.java", + }, 600) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedObjectUploadPart(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("上传分片", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectUploadPart(PresignedObjectUploadPart{ + UserID: 1, + ObjectID: 7, + Index: 3, + }, 600) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + +func Test_PresignedCompleteMultipartUpload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("合并分片", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectCompleteMultipartUpload(PresignedObjectCompleteMultipartUpload{ + UserID: 1, + ObjectID: 7, + Indexes: []int{1, 2, 3}, + }, 600) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} diff --git a/sdks/storage/cdsapi/signer.go b/sdks/storage/cdsapi/signer.go new file mode 100644 index 0000000..67e39e5 --- /dev/null +++ b/sdks/storage/cdsapi/signer.go @@ -0,0 +1,114 @@ +package cdsapi + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "time" + + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" +) + +const ( + AuthService = "jcs" + AuthRegion = "any" +) + +// 对一个请求进行签名，并将签名信息添加到请求头中。 +// +// 会读取请求体计算sha256哈希值。如果hash值已知，可以使用SignWithPayloadHash方法。 +func Sign(req *http.Request, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + payloadHash := "" + if req.Body != nil { + data, err := io.ReadAll(req.Body) + if err != nil { + return err + } + req.Body.Close() + req.Body = io.NopCloser(bytes.NewReader(data)) + + hasher := sha256.New() + hasher.Write(data) + payloadHash = hex.EncodeToString(hasher.Sum(nil)) + } else { + hash := sha256.Sum256([]byte("")) + payloadHash = hex.EncodeToString(hash[:]) + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, payloadHash, AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 对一个请求进行签名，不计算请求体的哈希，适合上传文件接口。 +func SignWithoutBody(req *http.Request, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 对一个请求进行签名，签名时使用指定的哈希值作为请求体的哈希值。 +// +// 参数payloadHash必须为sha256哈希值的16进制字符串，全小写。 +func SignWithPayloadHash(req *http.Request, payloadHash string, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, payloadHash, AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 生成一个带签名的URL。 +// +// expiration为签名过期时间，单位为秒。 +// +// 签名时不会包含请求体的哈希值。注：不要设置任何额外的Header（除了自动添加的Host），以免签名校验不通过 +func Presign(req *http.Request, accessKey, secretKey string, expiration int) (string, error) { + query := req.URL.Query() + query.Add("X-Expires", fmt.Sprintf("%v", expiration)) + + req.URL.RawQuery = query.Encode() + + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return "", err + } + + signer := v4.NewSigner() + signedURL, _, err := signer.PresignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + return signedURL, err +} diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index 3050f0d..68c1f00 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -1,14 +1,10 @@ package cdsapi import ( - "fmt" - "net/url" - "strings" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" - "gitlink.org.cn/cloudream/common/utils/serder" ) const StorageLoadPackagePath = "/storage/loadPackage" @@ -17,37 +13,21 @@ type StorageLoadPackageReq struct { UserID cdssdk.UserID `json:"userID" binding:"required"` PackageID cdssdk.PackageID `json:"packageID" binding:"required"` StorageID cdssdk.StorageID `json:"storageID" binding:"required"` + RootPath string `json:"rootPath"` } -type StorageLoadPackageResp struct { - FullPath string `json:"fullPath"` // TODO 临时保留给中期测试的前端使用，后续会删除 - PackagePath string `json:"packagePath"` - LocalBase string `json:"localBase"` - RemoteBase string `json:"remoteBase"` -} - -func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPackageResp, error) { - url, err := url.JoinPath(c.baseURL, StorageLoadPackagePath) - if err != nil { - return nil, err - } - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } +func (r *StorageLoadPackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, StorageLoadPackagePath, r) +} - codeResp, err := ParseJSONResponse[response[StorageLoadPackageResp]](resp) - if err != nil { - return nil, err - } +type StorageLoadPackageResp struct{} - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *StorageLoadPackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPackageResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageLoadPackageResp{}) } const StorageCreatePackagePath = "/storage/createPackage" @@ -61,71 +41,63 @@ type StorageCreatePackageReq struct { StorageAffinity cdssdk.StorageID `json:"storageAffinity"` } -type StorageCreatePackageResp struct { - PackageID cdssdk.PackageID `json:"packageID"` +func (r *StorageCreatePackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, StorageCreatePackagePath, r) } -func (c *Client) StorageCreatePackage(req StorageCreatePackageReq) (*StorageCreatePackageResp, error) { - url, err := url.JoinPath(c.baseURL, StorageCreatePackagePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - contType := resp.Header.Get("Content-Type") - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[StorageCreatePackageResp] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return nil, fmt.Errorf("parsing response: %w", err) - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +type StorageCreatePackageResp struct { + Package cdssdk.Package `json:"package"` +} - return nil, codeResp.ToError() - } +func (r *StorageCreatePackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, fmt.Errorf("unknow response content type: %s", contType) +func (c *Client) StorageCreatePackage(req StorageCreatePackageReq) (*StorageCreatePackageResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageCreatePackageResp{}) } const StorageGetPath = "/storage/get" type StorageGet struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - StorageID cdssdk.StorageID `form:"storageID" json:"storageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + StorageID cdssdk.StorageID `form:"storageID" url:"storageID" binding:"required"` +} + +func (r *StorageGet) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, StorageGetPath, r) } + type StorageGetResp struct { cdssdk.Storage } +func (r *StorageGetResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) StorageGet(req StorageGet) (*StorageGetResp, error) { - url, err := url.JoinPath(c.baseURL, StorageGetPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[StorageGetResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageGetResp{}) +} + +const StorageDeleteFilesPath = "/v1/storage/deleteFiles" + +type StorageDeleteFiles struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` + StorageID cdssdk.StorageID `json:"storageID" binding:"required"` + Pathes []string `json:"pathes"` +} + +func (r *StorageDeleteFiles) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, StorageDeleteFilesPath, r) +} + +type StorageDeleteFilesResp struct{} + +func (r *StorageDeleteFilesResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *Client) StorageDeleteFiles(req StorageDeleteFiles) (*StorageDeleteFilesResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageDeleteFilesResp{}) } diff --git a/sdks/storage/cdsapi/storage_test.go b/sdks/storage/cdsapi/storage_test.go index e12f984..c61ad46 100644 --- a/sdks/storage/cdsapi/storage_test.go +++ b/sdks/storage/cdsapi/storage_test.go @@ -131,7 +131,7 @@ func Test_ObjectList(t *testing.T) { URL: "http://localhost:7890", }) - resp, err := cli.Object().List(ObjectList{ + resp, err := cli.Object().ListByPath(ObjectListByPath{ UserID: 1, PackageID: 10, Path: "100x100K/zexema", @@ -246,23 +246,58 @@ func Test_Cache(t *testing.T) { }) } -func Test_GetHubInfos(t *testing.T) { - Convey("测试获取hub信息", t, func() { +func Test_Sign(t *testing.T) { + Convey("签名接口", t, func() { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890/v1", + AccessKey: "123456", + SecretKey: "123456", + }) + + fileData := make([]byte, 4096) + for i := 0; i < len(fileData); i++ { + fileData[i] = byte(i) + } + + pkgName := uuid.NewString() + createResp, err := cli.Package().Create(PackageCreate{ + UserID: 1, + BucketID: 1, + Name: pkgName, }) - resp1, err := cli.Package().GetCachedStorages(PackageGetCachedStoragesReq{ - PackageID: 11, + So(err, ShouldBeNil) + + _, err = cli.Object().Upload(ObjectUpload{ + ObjectUploadInfo: ObjectUploadInfo{ + UserID: 1, + PackageID: createResp.Package.PackageID, + }, + Files: iterator.Array( + &UploadingObject{ + Path: "abc/test", + File: io.NopCloser(bytes.NewBuffer(fileData)), + }, + &UploadingObject{ + Path: "test4", + File: io.NopCloser(bytes.NewBuffer(fileData)), + }, + ), + }) + So(err, ShouldBeNil) + + getResp, err := cli.Package().Get(PackageGetReq{ UserID: 1, + PackageID: createResp.Package.PackageID, }) So(err, ShouldBeNil) - fmt.Printf("resp1: %v\n", resp1) - resp2, err := cli.Package().GetLoadedStorages(PackageGetLoadedStoragesReq{ - PackageID: 11, + So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID) + So(getResp.Package.Name, ShouldEqual, pkgName) + + err = cli.Package().Delete(PackageDelete{ UserID: 1, + PackageID: createResp.Package.PackageID, }) So(err, ShouldBeNil) - fmt.Printf("resp2: %v\n", resp2) }) } diff --git a/sdks/storage/cdsapi/user.go b/sdks/storage/cdsapi/user.go new file mode 100644 index 0000000..01c8870 --- /dev/null +++ b/sdks/storage/cdsapi/user.go @@ -0,0 +1,72 @@ +package cdsapi + +import ( + "net/http" + + "gitlink.org.cn/cloudream/common/sdks" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" +) + +const UserCreatePath = "/v1/user/create" + +type UserCreate struct { + Name string `json:"name" binding:"required"` +} + +func (r *UserCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, UserCreatePath, r) +} + +type UserCreateResp struct { + User cdssdk.User `json:"user"` +} + +func (r *UserCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, req, &UserCreateResp{}) +} + +const UserDeletePath = "/v1/user/delete" + +type UserDelete struct { + UserID cdssdk.UserID `json:"userID" binding:"required"` +} + +func (r *UserDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, UserDeletePath, r) +} + +type UserDeleteResp struct{} + +func (r *UserDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *Client) UserDelete(req *UserDelete) error { + return JSONAPINoData(c.cfg, http.DefaultClient, req) +} + +const UserBatchGetStatsPath = "/v1/user/batchGetStats" + +type UserBatchGetStats struct { + UserIDs []cdssdk.UserID `json:"userIDs" binding:"required"` +} + +func (r *UserBatchGetStats) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, UserBatchGetStatsPath, r) +} + +type UserBatchGetStatsResp struct { + Stats []*cdssdk.UserStats `json:"stats"` +} + +func (r *UserBatchGetStatsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + +func (c *Client) UserBatchGetStats(req *UserBatchGetStats) (*UserBatchGetStatsResp, error) { + return JSONAPI(c.cfg, http.DefaultClient, req, &UserBatchGetStatsResp{}) +} diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 2ef1bc8..811c225 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -1,12 +1,19 @@ package cdsapi import ( + "crypto/sha256" + "encoding/hex" "fmt" "io" + "mime/multipart" "net/http" + ul "net/url" "path/filepath" "strings" + "github.com/google/go-querystring/query" + "gitlink.org.cn/cloudream/common/pkgs/iterator" + "gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -36,3 +43,155 @@ func ParseJSONResponse[TBody any](resp *http.Response) (TBody, error) { return ret, fmt.Errorf("unknow response content type: %s, status: %d, body(prefix): %s", contType, resp.StatusCode, strCont[:math2.Min(len(strCont), 200)]) } + +func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req, resp Resp) (Resp, error) { + + param := req.MakeParam() + + httpReq, err := param.MakeRequest(cfg.URL) + if err != nil { + return resp, err + } + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + err = SignWithPayloadHash(httpReq, calcSha256(param.Body), cfg.AccessKey, cfg.SecretKey) + if err != nil { + return resp, err + } + } + + httpResp, err := cli.Do(httpReq) + if err != nil { + return resp, err + } + + err = resp.ParseResponse(httpResp) + return resp, err +} + +func JSONAPINoData[Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) error { + param := req.MakeParam() + + httpReq, err := param.MakeRequest(cfg.URL) + if err != nil { + return err + } + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + err = SignWithPayloadHash(httpReq, calcSha256(param.Body), cfg.AccessKey, cfg.SecretKey) + if err != nil { + return err + } + } + + resp, err := cli.Do(httpReq) + if err != nil { + return err + } + + return sdks.ParseCodeDataJSONResponse(resp, any(nil)) +} + +func calcSha256(body sdks.RequestBody) string { + hasher := sha256.New() + switch body := body.(type) { + case *sdks.StringBody: + hasher.Write([]byte(body.Value)) + return hex.EncodeToString(hasher.Sum(nil)) + + case *sdks.BytesBody: + hasher.Write(body.Value) + return hex.EncodeToString(hasher.Sum(nil)) + + case *sdks.StreamBody: + return "" + + default: + hash := sha256.Sum256([]byte("")) + return hex.EncodeToString(hash[:]) + } +} + +func PostMultiPart(cfg *Config, url string, info any, files http2.MultiPartFileIterator) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return nil, err + } + + pr, pw := io.Pipe() + muWriter := multipart.NewWriter(pw) + + req.Header.Set("Content-Type", fmt.Sprintf("%s;boundary=%s", http2.ContentTypeMultiPart, muWriter.Boundary())) + + writeResult := make(chan error, 1) + go func() { + writeResult <- func() error { + defer pw.Close() + defer muWriter.Close() + + if info != nil { + mp, err := query.Values(info) + if err != nil { + return fmt.Errorf("formValues object to map failed, err: %w", err) + } + + for k, v := range mp { + err := muWriter.WriteField(k, v[0]) + if err != nil { + return fmt.Errorf("write form field failed, err: %w", err) + } + } + } + + for { + file, err := files.MoveNext() + if err == iterator.ErrNoMoreItem { + break + } + if err != nil { + return fmt.Errorf("opening file: %w", err) + } + + err = sendFileOnePart(muWriter, file.FieldName, file.FileName, file.File) + file.File.Close() + if err != nil { + return err + } + } + + return nil + }() + }() + + req.Body = pr + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + err = SignWithoutBody(req, cfg.AccessKey, cfg.SecretKey) + if err != nil { + return nil, err + } + } + + cli := http.Client{} + resp, err := cli.Do(req) + if err != nil { + return nil, err + } + + writeErr := <-writeResult + if writeErr != nil { + return nil, writeErr + } + + return resp, nil +} + +func sendFileOnePart(muWriter *multipart.Writer, fieldName, fileName string, file io.ReadCloser) error { + w, err := muWriter.CreateFormFile(fieldName, ul.PathEscape(fileName)) + if err != nil { + return fmt.Errorf("create form file failed, err: %w", err) + } + + _, err = io.Copy(w, file) + return err +} diff --git a/sdks/storage/filehash.go b/sdks/storage/filehash.go index 9fecbaf..b60cc30 100644 --- a/sdks/storage/filehash.go +++ b/sdks/storage/filehash.go @@ -20,6 +20,7 @@ type FileHash string const ( FullHashPrefix = "Full" CompositeHashPrefix = "Comp" + EmptyHash = FileHash("Full0000000000000000000000000000000000000000000000000000000000000000") ) func (h *FileHash) GetPrefix() string { @@ -30,6 +31,12 @@ func (h *FileHash) GetHash() string { return string((*h)[4:]) } +// 由调用者保证Hash值有效 +func (h *FileHash) GetHashBytes() []byte { + bytes, _ := hex.DecodeString(h.GetHash()) + return bytes +} + func (h *FileHash) GetHashPrefix(len int) string { return string((*h)[4 : 4+len]) } @@ -70,6 +77,10 @@ func NewFullHash(hash []byte) FileHash { return FileHash(FullHashPrefix + strings.ToUpper(hex.EncodeToString(hash))) } +func NewFullHashFromString(hashStr string) FileHash { + return FileHash(FullHashPrefix + strings.ToUpper(hashStr)) +} + func CalculateCompositeHash(segmentHashes [][]byte) FileHash { data := make([]byte, len(segmentHashes)*32) for i, segmentHash := range segmentHashes { diff --git a/sdks/storage/models.go b/sdks/storage/models.go index 05df652..e1c622f 100644 --- a/sdks/storage/models.go +++ b/sdks/storage/models.go @@ -40,6 +40,7 @@ var RedundancyUnion = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTyp (*ECRedundancy)(nil), (*LRCRedundancy)(nil), (*SegmentRedundancy)(nil), + (*MultipartUploadRedundancy)(nil), )), "type") type NoneRedundancy struct { @@ -229,16 +230,38 @@ func (b *SegmentRedundancy) CalcSegmentRange(start int64, end *int64) (segIdxSta return } +type MultipartUploadRedundancy struct { + serder.Metadata `union:"multipartUpload"` + Type string `json:"type"` +} + +func NewMultipartUploadRedundancy() *MultipartUploadRedundancy { + return &MultipartUploadRedundancy{ + Type: "multipartUpload", + } +} + +type User struct { + UserID UserID `gorm:"column:UserID; primaryKey; type:bigint" json:"userID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + Password string `gorm:"column:Password; type:varchar(255); not null" json:"password"` +} + +func (User) TableName() string { + return "User" +} + const ( PackageStateNormal = "Normal" PackageStateDeleted = "Deleted" ) type Package struct { - PackageID PackageID `gorm:"column:PackageID; primaryKey; type:bigint; autoIncrement" json:"packageID"` - Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` - BucketID BucketID `gorm:"column:BucketID; type:bigint; not null" json:"bucketID"` - State string `gorm:"column:State; type:varchar(255); not null" json:"state"` + PackageID PackageID `gorm:"column:PackageID; primaryKey; type:bigint; autoIncrement" json:"packageID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + BucketID BucketID `gorm:"column:BucketID; type:bigint; not null" json:"bucketID"` + CreateTime time.Time `gorm:"column:CreateTime; type:datetime; not null" json:"createTime"` + State string `gorm:"column:State; type:varchar(255); not null" json:"state"` } func (Package) TableName() string { @@ -315,9 +338,10 @@ func (PinnedObject) TableName() string { } type Bucket struct { - BucketID BucketID `gorm:"column:BucketID; primaryKey; type:bigint; autoIncrement" json:"bucketID"` - Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` - CreatorID UserID `gorm:"column:CreatorID; type:bigint; not null" json:"creatorID"` + BucketID BucketID `gorm:"column:BucketID; primaryKey; type:bigint; autoIncrement" json:"bucketID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + CreateTime time.Time `gorm:"column:CreateTime; type:datetime; not null" json:"createTime"` + CreatorID UserID `gorm:"column:CreatorID; type:bigint; not null" json:"creatorID"` } func (Bucket) TableName() string { @@ -361,3 +385,14 @@ type CodeError struct { func (e *CodeError) Error() string { return fmt.Sprintf("code: %s, message: %s", e.Code, e.Message) } + +type PackageHash struct { + // 16进制字符串格式的sha256哈希值 + Hash string `json:"hash"` +} + +type UserStats struct { + UserID UserID `json:"userID" gorm:"column:UserID"` + FileCount int64 `json:"fileCount" gorm:"column:FileCount"` + TotalSize int64 `json:"totalSize" gorm:"column:TotalSize"` +} diff --git a/sdks/storage/public_storage.go b/sdks/storage/public_storage.go new file mode 100644 index 0000000..761a6a8 --- /dev/null +++ b/sdks/storage/public_storage.go @@ -0,0 +1,48 @@ +package cdssdk + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/utils/serder" +) + +type PublicStoreConfig interface { + GetPublicStoreType() string + // 输出调试用的字符串，不要包含敏感信息 + String() string +} + +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[PublicStoreConfig]( + (*LocalPublicStorage)(nil), + (*S3PublicStorage)(nil), +)), "type") + +type LocalPublicStorage struct { + serder.Metadata `union:"Local"` + Type string `json:"type"` + // 调度Package时的Package的根路径 + LoadBase string `json:"loadBase"` +} + +func (s *LocalPublicStorage) GetPublicStoreType() string { + return "Local" +} + +func (s *LocalPublicStorage) String() string { + return fmt.Sprintf("Local[LoadBase=%v]", s.LoadBase) +} + +type S3PublicStorage struct { + serder.Metadata `union:"S3"` + Type string `json:"type"` + LoadBase string `json:"loadBase"` +} + +func (s *S3PublicStorage) GetPublicStoreType() string { + return "S3" +} + +func (s *S3PublicStorage) String() string { + return fmt.Sprintf("S3[LoadBase=%v]", s.LoadBase) +} diff --git a/sdks/storage/shared_storage.go b/sdks/storage/shared_storage.go deleted file mode 100644 index 7364143..0000000 --- a/sdks/storage/shared_storage.go +++ /dev/null @@ -1,33 +0,0 @@ -package cdssdk - -import ( - "fmt" - - "gitlink.org.cn/cloudream/common/pkgs/types" - "gitlink.org.cn/cloudream/common/utils/serder" -) - -type SharedStoreConfig interface { - GetSharedStoreType() string - // 输出调试用的字符串，不要包含敏感信息 - String() string -} - -var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[SharedStoreConfig]( - (*LocalSharedStorage)(nil), -)), "type") - -type LocalSharedStorage struct { - serder.Metadata `union:"Local"` - Type string `json:"type"` - // 调度Package时的Package的根路径 - LoadBase string `json:"loadBase"` -} - -func (s *LocalSharedStorage) GetSharedStoreType() string { - return "Local" -} - -func (s *LocalSharedStorage) String() string { - return fmt.Sprintf("Local[LoadBase=%v]", s.LoadBase) -} diff --git a/sdks/storage/storage.go b/sdks/storage/storage.go index 0d1fcd8..9f1fffd 100644 --- a/sdks/storage/storage.go +++ b/sdks/storage/storage.go @@ -17,8 +17,7 @@ type Storage struct { // 分片存储服务的配置数据 ShardStore ShardStoreConfig `json:"shardStore" gorm:"column:ShardStore; type:json; serializer:union"` // 共享存储服务的配置数据 - SharedStore SharedStoreConfig `json:"sharedStore" gorm:"column:SharedStore; type:json; serializer:union"` - // SharedStore + PublicStore PublicStoreConfig `json:"publicStore" gorm:"column:PublicStore; type:json; serializer:union"` // 存储服务拥有的特别功能 Features []StorageFeature `json:"features" gorm:"column:Features; type:json; serializer:union"` } @@ -39,12 +38,31 @@ type StorageType interface { } var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[StorageType]( + (*MashupStorageType)(nil), (*LocalStorageType)(nil), (*OBSType)(nil), (*OSSType)(nil), (*COSType)(nil), + (*EFileType)(nil), + (*S3Type)(nil), )), "type") +// 多种存储服务的混合存储服务。需谨慎选择存储服务的组合，避免出Bug +type MashupStorageType struct { + serder.Metadata `union:"Mashup"` + Type string `json:"type"` + Agent StorageType `json:"agent"` // 创建Agent时，使用的存储服务类型 + Feature StorageType `json:"feature"` // 根据Feature创建组件时使用的存储服务类型 +} + +func (a *MashupStorageType) GetStorageType() string { + return "Mashup" +} + +func (a *MashupStorageType) String() string { + return "Mashup" +} + type LocalStorageType struct { serder.Metadata `union:"Local"` Type string `json:"type"` @@ -84,6 +102,7 @@ type OBSType struct { SK string `json:"secretAccessKey"` Endpoint string `json:"endpoint"` Bucket string `json:"bucket"` + ProjectID string `json:"projectID"` } func (a *OBSType) GetStorageType() string { @@ -111,3 +130,42 @@ func (a *COSType) GetStorageType() string { func (a *COSType) String() string { return "COS" } + +type EFileType struct { + serder.Metadata `union:"EFile"` + Type string `json:"type"` + TokenURL string `json:"tokenURL"` + APIURL string `json:"apiURL"` + TokenExpire int `json:"tokenExpire"` // 单位秒 + User string `json:"user"` + Password string `json:"password"` + OrgID string `json:"orgID"` + ClusterID string `json:"clusterID"` +} + +func (a *EFileType) GetStorageType() string { + return "EFile" +} + +func (a *EFileType) String() string { + return "EFile" +} + +// 通用的S3协议的存储服务 +type S3Type struct { + serder.Metadata `union:"S3"` + Type string `json:"type"` + Region string `json:"region"` + AK string `json:"accessKeyId"` + SK string `json:"secretAccessKey"` + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` +} + +func (a *S3Type) GetStorageType() string { + return "S3" +} + +func (a *S3Type) String() string { + return "S3" +} diff --git a/sdks/storage/storage_feature.go b/sdks/storage/storage_feature.go index 0e150cb..3e97cda 100644 --- a/sdks/storage/storage_feature.go +++ b/sdks/storage/storage_feature.go @@ -17,6 +17,8 @@ var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Storage (*BypassWriteFeature)(nil), (*MultipartUploadFeature)(nil), (*InternalServerlessCallFeature)(nil), + (*S2STransferFeature)(nil), + (*ECMultiplierFeature)(nil), )), "type") type TempStore struct { @@ -78,3 +80,33 @@ func (f *InternalServerlessCallFeature) GetFeatureType() string { func (f *InternalServerlessCallFeature) String() string { return "InternalServerlessCall" } + +// 存储服务之间直传文件 +type S2STransferFeature struct { + serder.Metadata `union:"S2STransfer"` + Type string `json:"type"` + TempDir string `json:"tempDir"` // 临时文件存放目录 +} + +func (f *S2STransferFeature) GetFeatureType() string { + return "S2STransfer" +} + +func (f *S2STransferFeature) String() string { + return "S2STransfer" +} + +// 存储服务提供了能进行EC计算的接口 +type ECMultiplierFeature struct { + serder.Metadata `union:"ECMultiplier"` + Type string `json:"type"` + TempDir string `json:"tempDir"` // 临时文件存放目录 +} + +func (f *ECMultiplierFeature) GetFeatureType() string { + return "ECMultiplier" +} + +func (f *ECMultiplierFeature) String() string { + return "ECMultiplier" +} diff --git a/sdks/storage/utils.go b/sdks/storage/utils.go new file mode 100644 index 0000000..94c6813 --- /dev/null +++ b/sdks/storage/utils.go @@ -0,0 +1,18 @@ +package cdssdk + +import ( + "strings" +) + +func JoinObjectPath(comps ...string) string { + return strings.Join(comps, ObjectPathSeparator) +} + +func SplitObjectPath(pat string) []string { + return strings.Split(pat, ObjectPathSeparator) +} + +func BaseName(pat string) string { + idx := strings.LastIndex(pat, ObjectPathSeparator) + return pat[idx+1:] +} diff --git a/sdks/uploader/client.go b/sdks/uploader/client.go new file mode 100644 index 0000000..7b1f385 --- /dev/null +++ b/sdks/uploader/client.go @@ -0,0 +1,57 @@ +package uploadersdk + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/uploader/config.go b/sdks/uploader/config.go new file mode 100644 index 0000000..71cf23d --- /dev/null +++ b/sdks/uploader/config.go @@ -0,0 +1,5 @@ +package uploadersdk + +type Config struct { + URL string `json:"url"` +} diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go new file mode 100644 index 0000000..9ad3f8d --- /dev/null +++ b/sdks/uploader/models.go @@ -0,0 +1,236 @@ +package uploadersdk + +import ( + "gitlink.org.cn/cloudream/common/pkgs/types" + sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/serder" + "time" +) + +type ClusterID string + +type BlockChain struct { + ObjectID cdssdk.ObjectID `gorm:"column:object_id" json:"objectID"` + BlockChainID string `gorm:"column:blockchain_id" json:"blockChainID"` + BlockChainType string `gorm:"column:blockchain_type" json:"blockChainType"` + //FileName string `gorm:"column:file_name" json:"fileName"` + //FileHash string `gorm:"column:file_hash" json:"fileHash"` + //FileSize int64 `gorm:"column:file_size" json:"fileSize"` +} + +func (BlockChain) TableName() string { + return "block_chain" // 确保和数据库中的表名一致 +} + +type Binding struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` +} + +type BindingDAO struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + BindingCluster []BindingCluster `gorm:"foreignKey:binding_id;references:id" json:"bindingCluster"` +} + +type BindingAccessData struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + UserName string `gorm:"column:username" json:"userName"` + SSOId string `gorm:"column:sso_id" json:"ssoID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + ApplicantID cdssdk.UserID `gorm:"column:applicant_id" json:"applicantID"` + Status string `gorm:"column:status" json:"status"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` +} + +type BindingDetail struct { + ID DataID `json:"ID"` + UserID cdssdk.UserID `json:"ownerID"` + UserName string `json:"userName"` + SSOId string `json:"ssoID"` + Name string `json:"Name"` + Info sch.DataBinding `json:"info"` + Package Package `json:"package"` + Status string `json:"status"` + AccessLevel string `json:"accessLevel"` + CreateTime time.Time `json:"createTime"` +} + +func (Binding) TableName() string { + return "bindings" // 确保和数据库中的表名一致 +} + +type BindingCluster struct { + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + ClusterID ClusterID `gorm:"column:cluster_id" json:"clusterID"` + Status string `gorm:"column:status" json:"status"` + Param string `gorm:"column:param" json:"Param"` + JsonData string `gorm:"column:json_data" json:"jsonData"` +} + +func (BindingCluster) TableName() string { + return "binding_cluster" // 确保和数据库中的表名一致 +} + +type Folder struct { + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + Path string `gorm:"column:path_name" json:"path"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` +} + +func (Folder) TableName() string { + return "folders" +} + +type DataID int64 +type FolderID int64 + +type Cluster struct { + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"PackageID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + StorageID cdssdk.StorageID `gorm:"column:storage_id" json:"storageID"` +} + +func (Cluster) TableName() string { + return "uploaded_cluster" // 确保和数据库中的表名一致 +} + +type Package struct { + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` + Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` + UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` + Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` + UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` + BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` + PackageType string `gorm:"column:package_type" json:"packageType"` +} + +type PackageDAO struct { + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` + UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 + Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` + UploadPriority string `gorm:"column:upload_priority" json:"uploadPriority"` + Param string `gorm:"column:param" json:"param"` + PackageType string `gorm:"column:package_type" json:"packageType"` +} + +type PackageCloneDAO struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + ParentPackageID cdssdk.PackageID `gorm:"column:parent_package_id" json:"parentPackageID"` + ClonePackageID cdssdk.PackageID `gorm:"column:clone_package_id" json:"clonePackageID"` + Name string `gorm:"column:name" json:"name"` + Description string `gorm:"column:description" json:"description"` + BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + //ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` +} + +func (PackageCloneDAO) TableName() string { + return "package_clone" // 确保和数据库中的表名一致 +} + +type PackageCloneParam struct { + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + PackageName string `json:"packageName" binding:"required"` + Name string `json:"name"` + Description string `json:"description"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` + ImageID schsdk.ImageID `json:"imageID"` +} + +type PackageCloneVO struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + ParentPackageID cdssdk.PackageID `gorm:"column:parent_package_id" json:"parentPackageID"` + ClonePackageID cdssdk.PackageID `gorm:"column:clone_package_id" json:"clonePackageID"` + Name string `gorm:"column:name" json:"name"` + Description string `gorm:"column:description" json:"description"` + BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + //ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` + ImageID string `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"` +} + +type ClusterMapping struct { + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + ClusterName string `gorm:"column:cluster_name" json:"clusterName"` + StorageID cdssdk.StorageID `gorm:"column:storage_id" json:"storageID"` + StorageType string `gorm:"column:storage_type" json:"storageType"` + StoragePath string `gorm:"column:storage_path" json:"storagePath"` +} + +func (ClusterMapping) TableName() string { + return "cluster_mapping" +} + +type ScheduleTarget interface { + Noop() +} + +var DataScheduleTargetTypeUnion = types.NewTypeUnion[ScheduleTarget]( + (*JCSScheduleTarget)(nil), + (*UrlScheduleTarget)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataScheduleTargetTypeUnion, "type") + +type ScheduleTargetBase struct{} + +func (d *ScheduleTargetBase) Noop() {} + +type JCSScheduleTarget struct { + ScheduleTargetBase + UserID cdssdk.UserID `json:"userID"` + ScheduleStorages []ScheduleStorage `json:"scheduleStorages"` +} + +type UrlScheduleTarget struct { + ScheduleTargetBase + ScheduleUrls []ScheduleUrl `json:"scheduleUrls"` +} + +type ScheduleUrl struct { + ClusterID ClusterID `json:"clusterID"` + //RepositoryName string `json:"repositoryName"` + JsonData string `json:"jsonData"` +} + +type ScheduleStorage struct { + StorageID cdssdk.StorageID `json:"storageID"` + RootPath string `json:"rootPath"` +} diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go new file mode 100644 index 0000000..6cae186 --- /dev/null +++ b/sdks/uploader/uploader.go @@ -0,0 +1,46 @@ +package uploadersdk + +import ( + "fmt" + "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "mime/multipart" + "net/url" + "strings" +) + +type ObjectUploadReq struct { + Info cdsapi.ObjectUploadInfo `form:"info" binding:"required"` + Files []*multipart.FileHeader `form:"files"` +} + +func (c *Client) Upload(req ObjectUploadReq) (*cdsapi.ObjectUploadResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/object/upload") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[cdsapi.ObjectUploadResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, codeResp.ToError() + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) +} diff --git a/utils/config/config.go b/utils/config/config.go index d1bdb89..ed98270 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -1,14 +1,23 @@ package config import ( - "encoding/json" "fmt" "os" "path/filepath" "github.com/imdario/mergo" + "gitlink.org.cn/cloudream/common/utils/serder/json" + "gitlink.org.cn/cloudream/common/utils/serder/json/exts" ) +var serder json.Serder + +func init() { + cfg := json.New() + cfg = cfg.UseExtension(exts.NewDuration()) + serder = cfg.Build() +} + // Load 从本地文件读取配置，加载配置文件 func Load(filePath string, cfg interface{}) error { fileData, err := os.ReadFile(filePath) @@ -17,7 +26,7 @@ func Load(filePath string, cfg interface{}) error { } // json.Unmarshal用于将JSON解码成结构体 - return json.Unmarshal(fileData, cfg) + return serder.Decode(fileData, cfg) } // DefaultLoad 默认的加载配置的方式： @@ -32,6 +41,7 @@ func DefaultLoad(modeulName string, defCfg interface{}) error { // TODO 可以考虑根据环境变量读取不同的配置 // filepath.Join用于将多个路径组合成一个路径 configFilePath := filepath.Join(filepath.Dir(execPath), "..", "confs", fmt.Sprintf("%s.config.json", modeulName)) + return Load(configFilePath, defCfg) } diff --git a/utils/http2/http.go b/utils/http2/http.go index ab67e21..f57b06f 100644 --- a/utils/http2/http.go +++ b/utils/http2/http.go @@ -119,6 +119,27 @@ func PostJSON(url string, param RequestParam) (*http.Response, error) { return defaultClient.Do(req) } +func PutJSON(url string, param RequestParam) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPut, url, nil) + if err != nil { + return nil, err + } + + if err = prepareQuery(req, param.Query); err != nil { + return nil, err + } + + if err = prepareHeader(req, param.Header); err != nil { + return nil, err + } + + if err = prepareJSONBody(req, param.Body); err != nil { + return nil, err + } + + return defaultClient.Do(req) +} + func PostForm(url string, param RequestParam) (*http.Response, error) { req, err := http.NewRequest(http.MethodPost, url, nil) if err != nil { diff --git a/utils/io2/chunked_split.go b/utils/io2/chunked_split.go index b6606c5..07893a0 100644 --- a/utils/io2/chunked_split.go +++ b/utils/io2/chunked_split.go @@ -32,7 +32,7 @@ func ChunkedSplit(stream io.Reader, chunkSize int, streamCount int, opts ...Chun loop: for { for i := 0; i < streamCount; i++ { - var rd int = 0 + var rd = 0 if !eof { var err error rd, err = io.ReadFull(stream, buf) diff --git a/utils/io2/io.go b/utils/io2/io.go index 8a0d113..da57b92 100644 --- a/utils/io2/io.go +++ b/utils/io2/io.go @@ -1,6 +1,9 @@ package io2 -import "io" +import ( + "io" + "sync" +) type PromiseWriteCloser[T any] interface { io.Writer @@ -35,6 +38,7 @@ type readCloserHook struct { callback func(closer io.ReadCloser) once int isBefore bool // callback调用时机，true则在closer的Close之前调用 + lock *sync.Mutex } func (hook *readCloserHook) Read(buf []byte) (n int, err error) { @@ -42,6 +46,9 @@ func (hook *readCloserHook) Read(buf []byte) (n int, err error) { } func (hook *readCloserHook) Close() error { + hook.lock.Lock() + defer hook.lock.Unlock() + if hook.once == onceDone { return hook.readCloser.Close() } @@ -69,6 +76,7 @@ func BeforeReadClosing(closer io.ReadCloser, callback func(closer io.ReadCloser) callback: callback, once: onceDisabled, isBefore: true, + lock: &sync.Mutex{}, } } @@ -78,6 +86,7 @@ func AfterReadClosed(closer io.ReadCloser, callback func(closer io.ReadCloser)) callback: callback, once: onceDisabled, isBefore: false, + lock: &sync.Mutex{}, } } @@ -87,16 +96,22 @@ func AfterReadClosedOnce(closer io.ReadCloser, callback func(closer io.ReadClose callback: callback, once: onceEnabled, isBefore: false, + lock: &sync.Mutex{}, } } type afterEOF struct { inner io.ReadCloser callback func(str io.ReadCloser, err error) + lock *sync.Mutex } func (hook *afterEOF) Read(buf []byte) (n int, err error) { n, err = hook.inner.Read(buf) + + hook.lock.Lock() + defer hook.lock.Unlock() + if hook.callback != nil { if err == io.EOF { hook.callback(hook.inner, nil) @@ -111,6 +126,10 @@ func (hook *afterEOF) Read(buf []byte) (n int, err error) { func (hook *afterEOF) Close() error { err := hook.inner.Close() + + hook.lock.Lock() + defer hook.lock.Unlock() + if hook.callback != nil { hook.callback(hook.inner, io.ErrClosedPipe) hook.callback = nil @@ -122,6 +141,7 @@ func AfterEOF(str io.ReadCloser, callback func(str io.ReadCloser, err error)) io return &afterEOF{ inner: str, callback: callback, + lock: &sync.Mutex{}, } } @@ -196,3 +216,17 @@ func DropWithBuf(str io.Reader, buf []byte) { } } } + +func ReadMost(str io.Reader, n int) ([]byte, error) { + buf := make([]byte, n) + n, err := io.ReadFull(str, buf) + if err == nil { + return buf, nil + } + + if err == io.EOF || err == io.ErrUnexpectedEOF { + return buf[:n], nil + } + + return buf[:n], err +} diff --git a/utils/io2/range.go b/utils/io2/range.go index a4a0b35..46b4add 100644 --- a/utils/io2/range.go +++ b/utils/io2/range.go @@ -37,16 +37,22 @@ func (r *rng) Read(p []byte) (n int, err error) { need := math2.Min(*r.length, int64(len(p))) rd, err := r.inner.Read(p[:need]) - if err != nil { - r.err = err - return rd, io.EOF - } *r.length -= int64(rd) if *r.length == 0 { r.err = io.EOF } + if err == nil { + return rd, nil + } + + if err != io.EOF { + r.err = err + return rd, err + } + + r.err = io.EOF return rd, nil } diff --git a/utils/io2/stats.go b/utils/io2/stats.go index b6c7956..cdc6ec5 100644 --- a/utils/io2/stats.go +++ b/utils/io2/stats.go @@ -2,21 +2,52 @@ package io2 import "io" -type Counter struct { +type counter struct { inner io.Reader count int64 } -func (c *Counter) Read(buf []byte) (n int, err error) { +func (c *counter) Read(buf []byte) (n int, err error) { n, err = c.inner.Read(buf) c.count += int64(n) return } -func (c *Counter) Count() int64 { +func (c *counter) Count() int64 { return c.count } -func NewCounter(inner io.Reader) *Counter { - return &Counter{inner: inner, count: 0} +func Counter(inner io.Reader) *counter { + return &counter{inner: inner, count: 0} +} + +type counterCloser struct { + inner io.ReadCloser + count int64 + callback func(cnt int64, err error) +} + +func (c *counterCloser) Read(buf []byte) (n int, err error) { + n, err = c.inner.Read(buf) + c.count += int64(n) + if err != nil && c.callback != nil { + c.callback(c.count, err) + c.callback = nil + } + return +} + +func (c *counterCloser) Close() error { + // 读取方主动Close，视为正常结束 + err := c.inner.Close() + if c.callback != nil { + c.callback(c.count, nil) + } + return err +} + +// 统计一个io.ReadCloser的读取字节数，在读取结束后调用callback函数。 +// 仅在读取方主动调用Close时，callback的err参数才会为nil。 +func CounterCloser(inner io.ReadCloser, callback func(cnt int64, err error)) io.ReadCloser { + return &counterCloser{inner: inner, count: 0, callback: callback} } diff --git a/utils/lo2/lo.go b/utils/lo2/lo.go index 930f6d3..41b794c 100644 --- a/utils/lo2/lo.go +++ b/utils/lo2/lo.go @@ -73,3 +73,24 @@ func Deref[T any](arr []*T) []T { return result } + +func AppendNew[T any](arr []T, items ...T) []T { + narr := make([]T, 0, len(arr)+len(items)) + narr = append(narr, arr...) + narr = append(narr, items...) + return narr +} + +func ArrayEquals[T comparable](arr1, arr2 []T) bool { + if len(arr1) != len(arr2) { + return false + } + + for i := 0; i < len(arr1); i++ { + if arr1[i] != arr2[i] { + return false + } + } + + return true +} diff --git a/utils/math2/math.go b/utils/math2/math.go index 7a8508b..d1bab7d 100644 --- a/utils/math2/math.go +++ b/utils/math2/math.go @@ -2,6 +2,18 @@ package math2 import "golang.org/x/exp/constraints" +func Sign[T constraints.Signed](v T) int { + if v > 0 { + return 1 + } + + if v < 0 { + return -1 + } + + return 0 +} + func Max[T constraints.Ordered](v1, v2 T) T { if v1 < v2 { return v2 @@ -72,3 +84,12 @@ func SplitN[T constraints.Integer](v T, n int) []T { return result } + +// 除法，如果除数为0，则返回默认值 +func DivOrDefault[T constraints.Integer | constraints.Float](v, d T, def T) T { + if d == 0 { + return def + } + + return v / d +} diff --git a/utils/math2/range.go b/utils/math2/range.go new file mode 100644 index 0000000..0f8ae2e --- /dev/null +++ b/utils/math2/range.go @@ -0,0 +1,99 @@ +package math2 + +type Range struct { + Offset int64 + Length *int64 +} + +// length为-1时Range.Length为nil +func NewRange(offset int64, length int64) Range { + if length == -1 { + return Range{Offset: offset, Length: nil} + } + return Range{Offset: offset, Length: &length} +} + +// 不包含end +func RangeFromStartEnd(start int64, end int64) Range { + length := end - start + return Range{Offset: start, Length: &length} +} + +// 给Length设置一个具体值 +func (r *Range) Fix(totalLen int64) { + len := totalLen - r.Offset + r.Length = &len +} + +// 如果Length为nil，则end为-1 +func (r *Range) ToStartEnd() (start int64, end int64) { + if r.Length == nil { + return r.Offset, -1 + } + + end = r.Offset + *r.Length + return r.Offset, end +} + +// 将范围限制在totalLen内。会同时设置Length的值 +func (r *Range) Clamp(totalLen int64) { + r.Offset = Min(r.Offset, totalLen) + if r.Length == nil { + len := totalLen - r.Offset + r.Length = &len + } else { + *r.Length = Min(*r.Length, totalLen-r.Offset) + } +} + +func (r *Range) Extend(other Range) { + newOffset := Min(r.Offset, other.Offset) + + if r.Length == nil { + r.Offset = newOffset + return + } + + if other.Length == nil { + r.Offset = newOffset + r.Length = nil + return + } + + otherEnd := other.Offset + *other.Length + rEnd := r.Offset + *r.Length + + newEnd := Max(otherEnd, rEnd) + r.Offset = newOffset + *r.Length = newEnd - newOffset +} + +func (r *Range) ExtendStart(start int64) { + r.Offset = Min(r.Offset, start) +} + +func (r *Range) ExtendEnd(end int64) { + if r.Length == nil { + return + } + + rEnd := r.Offset + *r.Length + newLen := Max(end, rEnd) - r.Offset + r.Length = &newLen +} + +func (r *Range) Equals(other Range) bool { + if r.Offset != other.Offset { + return false + } + + if r.Length == nil && other.Length == nil { + return true + } + + if r.Length == nil || other.Length == nil { + return false + } + + return *r.Length == *other.Length +} diff --git a/utils/reflect2/assign.go b/utils/reflect2/assign.go new file mode 100644 index 0000000..2e8d49d --- /dev/null +++ b/utils/reflect2/assign.go @@ -0,0 +1,29 @@ +package reflect2 + +import "reflect" + +// MergeNonZero 将 src 中非零值字段复制到 dst 中 +func MergeNonZero(dst, src any) { + dstVal := reflect.ValueOf(dst).Elem() + srcVal := reflect.ValueOf(src).Elem() + + for i := 0; i < dstVal.NumField(); i++ { + dstField := dstVal.Field(i) + srcField := srcVal.Field(i) + + // 如果 src 字段是零值，则跳过 + if isZero(srcField) { + continue + } + + // 如果 dst 字段可设置，才设置 + if dstField.CanSet() { + dstField.Set(srcField) + } + } +} + +func isZero(v reflect.Value) bool { + zero := reflect.Zero(v.Type()) + return reflect.DeepEqual(v.Interface(), zero.Interface()) +} diff --git a/utils/serder/json/exts/exts_test.go b/utils/serder/json/exts/exts_test.go new file mode 100644 index 0000000..348c1e2 --- /dev/null +++ b/utils/serder/json/exts/exts_test.go @@ -0,0 +1,35 @@ +package exts + +import ( + "testing" + "time" + + jsoniter "github.com/json-iterator/go" + . "github.com/smartystreets/goconvey/convey" +) + +func Test_Duration(t *testing.T) { + c := jsoniter.Config{} + api := c.Froze() + api.RegisterExtension(&DurationExt{}) + + Convey("Duration", t, func() { + type Test struct { + D1 time.Duration `json:"d1"` + D2 time.Duration `json:"d2"` + D3 *time.Duration `json:"d3"` + D4 *time.Duration `json:"d4"` + D5 *time.Duration `json:"d5"` + } + + var test Test + err := api.Unmarshal([]byte(`{"d1":"1h", "d2": null, "d3": "2h", "d4": null}`), &test) + So(err, ShouldBeNil) + So(test.D1, ShouldEqual, 1*time.Hour) + So(test.D2, ShouldEqual, 0) + So(*test.D3, ShouldEqual, 2*time.Hour) + So(test.D4, ShouldEqual, (*time.Duration)(nil)) + So(test.D5, ShouldEqual, (*time.Duration)(nil)) + }) + +} diff --git a/utils/serder/json/exts/time.go b/utils/serder/json/exts/time.go new file mode 100644 index 0000000..03ffb81 --- /dev/null +++ b/utils/serder/json/exts/time.go @@ -0,0 +1,76 @@ +package exts + +import ( + "fmt" + "time" + "unsafe" + + jsoniter "github.com/json-iterator/go" + "github.com/modern-go/reflect2" +) + +type DurationExt struct { + jsoniter.DummyExtension +} + +func NewDuration() *DurationExt { + return &DurationExt{} +} + +func (e *DurationExt) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { + if typ == reflect2.TypeOf(time.Duration(0)) { + return &DurationDecoder{ + isPointer: false, + } + } + + if typ == reflect2.TypeOf((*time.Duration)(nil)) { + return &DurationDecoder{ + isPointer: true, + } + } + + return nil +} + +type DurationDecoder struct { + isPointer bool +} + +func (e *DurationDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + nextType := iter.WhatIsNext() + if nextType == jsoniter.StringValue { + str := iter.ReadString() + dt, err := time.ParseDuration(str) + if err != nil { + iter.Error = err + return + } + + if e.isPointer { + p := (**time.Duration)(ptr) + *p = &dt + } else { + p := (*time.Duration)(ptr) + *p = dt + } + + } else if nextType == jsoniter.NilValue { + iter.ReadNil() + + if e.isPointer { + p := (**time.Duration)(ptr) + *p = nil + } else { + p := (*time.Duration)(ptr) + *p = time.Duration(0) + } + + } else { + iter.Error = fmt.Errorf("unsupported type %v", nextType) + return + } + +} + +var _ jsoniter.Extension = &DurationExt{} diff --git a/utils/serder/serder.go b/utils/serder/serder.go index 268215d..cbc7fbb 100644 --- a/utils/serder/serder.go +++ b/utils/serder/serder.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "reflect" + "strings" jsoniter "github.com/json-iterator/go" "github.com/mitchellh/mapstructure" @@ -65,6 +66,9 @@ func JSONToObjectExRaw(data []byte, ret any) error { } // 将JSON字符串转为对象。支持TypeUnion。 +// +// 如果发现反序列化后的结果不对，但没有返回错误，可以考虑是JSON字符串有问题， +// 尤其是在反序列化嵌套的TypeUnion时，如果内部的TypeUnion反序列化失败，错误是不会传递出来的（库的缺陷） func JSONToObjectStreamEx[T any](stream io.Reader) (T, error) { var ret T dec := defaultAPI.NewDecoder(stream) @@ -76,7 +80,19 @@ func JSONToObjectStreamEx[T any](stream io.Reader) (T, error) { return ret, nil } +func JSONToObjectStreamExRaw(stream io.Reader, ret any) error { + dec := defaultAPI.NewDecoder(stream) + err := dec.Decode(ret) + if err != nil { + return err + } + + return nil +} + // 将对象转为JSON字符串。如果需要支持解析TypeUnion类型，则使用"Ex"结尾的同名函数。 +// +// 注：[]byte会被base64编码，如果要JSON内容要给外部解析，那么应该避免使用[]byte。 func ObjectToJSON(obj any) ([]byte, error) { return json.Marshal(obj) } @@ -184,3 +200,79 @@ func ObjectToMap(obj any) (map[string]any, error) { } return mp, dec.Decode(obj) } + +// 1. 尝试解开所有引用 +// +// 2. nil值将会是空字符串 +func ObjectToMapString(obj any) (map[string]string, error) { + if obj == nil { + return make(map[string]string), nil + } + + v := reflect.ValueOf(obj) + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + if !v.IsValid() { + return make(map[string]string), nil + } + if v.Kind() != reflect.Struct { + return nil, fmt.Errorf("type %v is not a struct", v.Type()) + } + + mp := make(map[string]string) + objectToMapString(v, mp) + return mp, nil +} + +func objectToMapString(val reflect.Value, mp map[string]string) { + typ := val.Type() + for i := 0; i < val.NumField(); i++ { + vf := val.Field(i) + tf := typ.Field(i) + if tf.Anonymous { + objectToMapString(vf, mp) + continue + } + + fieldName := tf.Name + + omitEmpty := false + jsonTag := tf.Tag.Get("json") + if jsonTag != "" { + tagParts := strings.Split(jsonTag, ",") + fieldName = strings.TrimSpace(tagParts[0]) + if len(tagParts) > 1 { + for _, tagPart := range tagParts[1:] { + tagPart = strings.TrimSpace(tagPart) + if tagPart == "omitempty" { + omitEmpty = true + } + } + } + } + + for vf.Kind() == reflect.Ptr { + vf = vf.Elem() + } + + if !vf.IsValid() { + if omitEmpty { + continue + } + + mp[fieldName] = "" + continue + } + + if vf.IsZero() && omitEmpty { + continue + } + + if vf.Kind() == reflect.Array { + + } + + mp[fieldName] = fmt.Sprintf("%v", vf) + } +} diff --git a/utils/serder/serder_test.go b/utils/serder/serder_test.go index 4914531..e0d1f4c 100644 --- a/utils/serder/serder_test.go +++ b/utils/serder/serder_test.go @@ -591,7 +591,7 @@ func Test_ObjectToJSON2(t *testing.T) { union := types.NewTypeUnion[Base](&St1{}, St2{}) UseTypeUnionExternallyTagged(&union) - var val []Base = []Base{ + var val = []Base{ &St1{Val: "asd", Type: "St1"}, St2{Val: 123, Type: "St2"}, } @@ -607,7 +607,7 @@ func Test_ObjectToJSON2(t *testing.T) { union := types.NewTypeUnion[Base](&St1{}, St2{}) UseTypeUnionInternallyTagged(&union, "Type") - var val []Base = []Base{ + var val = []Base{ &St1{Val: "asd", Type: "St1"}, St2{Val: 123, Type: "St2"}, } @@ -623,7 +623,7 @@ func Test_ObjectToJSON2(t *testing.T) { union := types.NewTypeUnion[Base](&St1{}, St2{}) UseTypeUnionExternallyTagged(&union) - var val []Base = []Base{ + var val = []Base{ nil, } data, err := ObjectToJSONEx(val) @@ -638,7 +638,7 @@ func Test_ObjectToJSON2(t *testing.T) { union := types.NewTypeUnion[Base](&St1{}, St2{}) UseTypeUnionInternallyTagged(&union, "Type") - var val []Base = []Base{ + var val = []Base{ nil, } data, err := ObjectToJSONEx(val) @@ -648,4 +648,154 @@ func Test_ObjectToJSON2(t *testing.T) { So(err, ShouldBeNil) So(ret, ShouldResemble, val) }) + +} + +func Test_ObjectToJSON3(t *testing.T) { + Convey("反序列化TypeUnion时，JSON中对应字段类型不对", t, func() { + type Base interface{} + + union := types.NewTypeUnion[Base](&St1{}, &St2{}) + UseTypeUnionInternallyTagged(&union, "Type") + + v, err := JSONToObjectEx[[]Base]([]byte("[{\"Type\":\"St2\", \"Val\":[]}]")) + t.Logf("err: %v", err) + t.Logf("v: %+v", v[0]) + So(err, ShouldNotBeNil) + // So(ret, ShouldResemble, val) + }) +} + +type BaseCallback interface{} +type StCallback struct { + Metadata `union:"StCallback"` + Type string + Value string +} + +func (st *StCallback) OnUnionSerializing() { + st.Value = "called" + st.Type = "StCallback" +} + +func Test_ObjectToJSONEx4(t *testing.T) { + Convey("序列化Callback", t, func() { + union := types.NewTypeUnion[BaseCallback](&StCallback{}) + UseTypeUnionInternallyTagged(&union, "Type") + + val := []BaseCallback{&StCallback{}} + data, err := ObjectToJSONEx(val) + So(err, ShouldBeNil) + + ret, err := JSONToObjectEx[[]BaseCallback](data) + So(err, ShouldBeNil) + + So(len(ret), ShouldEqual, 1) + So(ret[0].(*StCallback).Value, ShouldEqual, "called") + }) +} + +type StStringer struct { +} + +func (s StStringer) String() string { + return "StStringer" +} + +func Test_ObjectToMapString(t *testing.T) { + Convey("结构体", t, func() { + type StEmb struct { + IntRef *int `json:"intRef,omitempty"` + BoolRef **bool `json:"boolRef"` + StrRef *string `json:"strRef"` + } + + type St struct { + StEmb + Int int + Bool bool + Str string + StStringer *StStringer + } + + st := St{ + StEmb: StEmb{ + IntRef: types.Ref(123), + BoolRef: types.Ref(types.Ref(true)), + }, + Int: 456, + Bool: false, + Str: "test", + StStringer: &StStringer{}, + } + + mp, err := ObjectToMapString(st) + So(err, ShouldBeNil) + + So(mp["intRef"], ShouldEqual, "123") + So(mp["boolRef"], ShouldEqual, "true") + So(mp["strRef"], ShouldEqual, "") + So(mp["Int"], ShouldEqual, "456") + So(mp["Bool"], ShouldEqual, "false") + So(mp["Str"], ShouldEqual, "test") + So(mp["StStringer"], ShouldEqual, "StStringer") + }) + + Convey("结构体引用", t, func() { + type StEmb struct { + IntRef *int `json:"intRef,omitempty"` + BoolRef **bool `json:"boolRef"` + StrRef *string `json:"strRef"` + } + + type St struct { + StEmb + Int int + Bool bool + Str string + StStringer *StStringer + } + + st := St{ + StEmb: StEmb{ + IntRef: types.Ref(123), + BoolRef: types.Ref(types.Ref(true)), + }, + Int: 456, + Bool: false, + Str: "test", + StStringer: &StStringer{}, + } + + mp, err := ObjectToMapString(&st) + So(err, ShouldBeNil) + + So(mp["intRef"], ShouldEqual, "123") + So(mp["boolRef"], ShouldEqual, "true") + So(mp["strRef"], ShouldEqual, "") + So(mp["Int"], ShouldEqual, "456") + So(mp["Bool"], ShouldEqual, "false") + So(mp["Str"], ShouldEqual, "test") + So(mp["StStringer"], ShouldEqual, "StStringer") + }) + + Convey("nil", t, func() { + mp, err := ObjectToMapString(nil) + So(err, ShouldBeNil) + So(mp, ShouldResemble, map[string]string{}) + }) + + Convey("nil指针", t, func() { + type St struct{} + var st *St + mp, err := ObjectToMapString(st) + So(err, ShouldBeNil) + So(mp, ShouldResemble, map[string]string{}) + }) + + Convey("非结构体", t, func() { + mp, err := ObjectToMapString(123) + So(err, ShouldNotBeNil) + So(mp, ShouldBeNil) + }) } diff --git a/utils/serder/types/types.go b/utils/serder/types/types.go index 0046926..a1cd975 100644 --- a/utils/serder/types/types.go +++ b/utils/serder/types/types.go @@ -1,3 +1,7 @@ package types type Metadata struct{} + +type OnUnionSerializing interface { + OnUnionSerializing() +} diff --git a/utils/serder/union_handler.go b/utils/serder/union_handler.go index ce4ac3e..dd6fb16 100644 --- a/utils/serder/union_handler.go +++ b/utils/serder/union_handler.go @@ -8,6 +8,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/modern-go/reflect2" "gitlink.org.cn/cloudream/common/pkgs/types" + sertypes "gitlink.org.cn/cloudream/common/utils/serder/types" ref2 "gitlink.org.cn/cloudream/common/utils/reflect2" ) @@ -211,6 +212,12 @@ func (e *InternallyTaggedEncoder) Encode(ptr unsafe.Pointer, stream *jsoniter.St val = reflect2.IFaceToEFace(ptr) } + if val != nil { + if on, ok := val.(sertypes.OnUnionSerializing); ok { + on.OnUnionSerializing() + } + } + // 可以考虑检查一下Type字段有没有赋值，没有赋值则将其赋值为union Tag指定的值 stream.WriteVal(val) } @@ -254,7 +261,7 @@ func (e *InternallyTaggedDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iter // 否则New出来的是会是**T，这将导致后续的反序列化出问题 if typ.Kind() == reflect.Pointer { val := reflect.New(typ.Elem()) - raw.ToVal(val.Interface()) + raw.ToVal(val.Interface()) // TODO 使用的库丢失了ToVal期间的错误信息，考虑换个库 retVal := reflect.NewAt(e.union.Union.UnionType, ptr) retVal.Elem().Set(val) @@ -292,6 +299,10 @@ func (e *ExternallyTaggedEncoder) Encode(ptr unsafe.Pointer, stream *jsoniter.St return } + if on, ok := val.(sertypes.OnUnionSerializing); ok { + on.OnUnionSerializing() + } + stream.WriteObjectStart() valType := ref2.TypeOfValue(val) if !e.union.Union.Include(valType) { diff --git a/utils/sync2/bucket_pool.go b/utils/sync2/bucket_pool.go new file mode 100644 index 0000000..d3b4114 --- /dev/null +++ b/utils/sync2/bucket_pool.go @@ -0,0 +1,95 @@ +package sync2 + +import "sync" + +type BucketPool[T any] struct { + empty []T + filled []T + emptyCond *sync.Cond + filledCond *sync.Cond + closed bool +} + +func NewBucketPool[T any]() *BucketPool[T] { + return &BucketPool[T]{ + emptyCond: sync.NewCond(&sync.Mutex{}), + filledCond: sync.NewCond(&sync.Mutex{}), + } +} + +func (p *BucketPool[T]) GetEmpty() (T, bool) { + p.emptyCond.L.Lock() + defer p.emptyCond.L.Unlock() + + if p.closed { + var t T + return t, false + } + + if len(p.empty) == 0 { + p.emptyCond.Wait() + } + + if len(p.empty) == 0 { + var t T + return t, false + } + + t := p.empty[0] + p.empty = p.empty[1:] + return t, true +} + +func (p *BucketPool[T]) PutEmpty(t T) { + p.emptyCond.L.Lock() + defer p.emptyCond.L.Unlock() + + p.empty = append(p.empty, t) + p.emptyCond.Signal() +} + +func (p *BucketPool[T]) GetFilled() (T, bool) { + p.filledCond.L.Lock() + defer p.filledCond.L.Unlock() + + if len(p.filled) == 0 { + if p.closed { + var t T + return t, false + } + + p.filledCond.Wait() + } + + if len(p.filled) == 0 { + var t T + return t, false + } + + t := p.filled[0] + p.filled = p.filled[1:] + return t, true +} + +func (p *BucketPool[T]) PutFilled(t T) { + p.filledCond.L.Lock() + defer p.filledCond.L.Unlock() + + p.filled = append(p.filled, t) + p.filledCond.Signal() +} + +func (p *BucketPool[T]) Close() { + // 在两个锁中分别关闭，防止变量可见性问题 + + p.emptyCond.L.Lock() + p.closed = true + p.emptyCond.L.Unlock() + + p.filledCond.L.Lock() + p.closed = true + p.filledCond.L.Unlock() + + p.emptyCond.Broadcast() + p.filledCond.Broadcast() +} diff --git a/utils/sync2/token_pool.go b/utils/sync2/token_pool.go new file mode 100644 index 0000000..3fda84b --- /dev/null +++ b/utils/sync2/token_pool.go @@ -0,0 +1,60 @@ +package sync2 + +import "sync" + +type TokenPool[T any] struct { + tokens []T + cond *sync.Cond + closed bool +} + +func NewTokenPool[T any]() *TokenPool[T] { + return &TokenPool[T]{ + tokens: make([]T, 0), + cond: sync.NewCond(new(sync.Mutex)), + } +} + +func NewTokenPoolWithTokens[T any](tokens ...T) *TokenPool[T] { + b := NewTokenPool[T]() + b.tokens = append(b.tokens, tokens...) + return b +} + +func (b *TokenPool[T]) Take() (T, bool) { + b.cond.L.Lock() + defer b.cond.L.Unlock() + var ret T + + if b.closed { + return ret, false + } + + if len(b.tokens) == 0 { + b.cond.Wait() + + if len(b.tokens) == 0 { + return ret, false + } + } + + ret = b.tokens[0] + b.tokens = b.tokens[1:] + + return ret, true +} + +func (b *TokenPool[T]) Put(t T) { + b.cond.L.Lock() + b.tokens = append(b.tokens, t) + b.cond.L.Unlock() + b.cond.Signal() +} + +func (b *TokenPool[T]) Close() { + b.cond.L.Lock() + b.closed = true + b.cond.L.Unlock() + + b.cond.Broadcast() +}