From 9c859bacffe744870f7240b89a2b3ad44245878f Mon Sep 17 00:00:00 2001 From: colorfulberry Date: Mon, 13 Jul 2020 16:49:02 +0800 Subject: [PATCH 01/17] feat: add routes for new and index page --- routers/repo/cloudbrain.go | 19 +++++++++++++++++++ routers/routes/routes.go | 5 +++++ templates/repo/cloudbrain/index.tmpl | 4 ++++ templates/repo/cloudbrain/new.tmpl | 4 ++++ 4 files changed, 32 insertions(+) create mode 100644 routers/repo/cloudbrain.go create mode 100644 templates/repo/cloudbrain/index.tmpl create mode 100644 templates/repo/cloudbrain/new.tmpl diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go new file mode 100644 index 000000000..94f4df2c6 --- /dev/null +++ b/routers/repo/cloudbrain.go @@ -0,0 +1,19 @@ +package repo + +import ( + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" +) + +const ( + tplCloudBrainIndex base.TplName = "repo/cloudbrain/index" + tplCloudBrainNew base.TplName = "repo/cloudbrain/new" +) + +func CloudBrainIndex(ctx *context.Context) { + ctx.HTML(200, tplCloudBrainIndex) +} + +func CloudBrainNew(ctx *context.Context) { + ctx.HTML(200, tplCloudBrainNew) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index ea9dd7f65..27c9e0c22 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -869,6 +869,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("", reqRepoDatasetWriter, bindIgnErr(auth.EditDatasetForm{}), repo.EditDatasetPost) }, context.RepoRef()) + m.Group("/cloudbrain", func() { + m.Get("", repo.CloudBrainIndex) + m.Get("/new", repo.CloudBrainNew) + }) + m.Group("/wiki", func() { m.Get("/?:page", repo.Wiki) m.Get("/_pages", repo.WikiPages) diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl new file mode 100644 index 000000000..97f9303bc --- /dev/null +++ b/templates/repo/cloudbrain/index.tmpl @@ -0,0 +1,4 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
\ No newline at end of file diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl new file mode 100644 index 000000000..97f9303bc --- /dev/null +++ b/templates/repo/cloudbrain/new.tmpl @@ -0,0 +1,4 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
\ No newline at end of file From daf35e238fec698a83e68c9eb7ff51830a652e9c Mon Sep 17 00:00:00 2001 From: colorfulberry Date: Mon, 13 Jul 2020 17:17:29 +0800 Subject: [PATCH 02/17] feat: improve the pages --- routers/repo/cloudbrain.go | 4 ++++ templates/repo/cloudbrain/index.tmpl | 3 ++- templates/repo/cloudbrain/new.tmpl | 3 ++- templates/repo/header.tmpl | 6 ++++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 94f4df2c6..a6aee1407 100644 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -11,9 +11,13 @@ const ( ) func CloudBrainIndex(ctx *context.Context) { + ctx.Data["PageIsViewCloudBrain"] = true + ctx.HTML(200, tplCloudBrainIndex) } func CloudBrainNew(ctx *context.Context) { + ctx.Data["PageIsViewCloudBrain"] = true + ctx.HTML(200, tplCloudBrainNew) } diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index 97f9303bc..399d17520 100644 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -1,4 +1,5 @@ {{template "base/head" .}}
{{template "repo/header" .}} -
\ No newline at end of file + +{{template "base/footer" .}} diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index 97f9303bc..399d17520 100644 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -1,4 +1,5 @@ {{template "base/head" .}}
{{template "repo/header" .}} -
\ No newline at end of file + +{{template "base/footer" .}} diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index 8478b1e7f..b0e546272 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -139,6 +139,12 @@ {{end}} + {{if .Permission.CanRead $.UnitTypeCode}} + + {{svg "octicon-circuit-board" 16}} {{.i18n.Tr "repo.cloudbrain"}} + + {{end}} + {{template "custom/extra_tabs" .}} {{if .Permission.IsAdmin}} From b0e6475069d4dd01369f39f07fa1bb1aa0bf395a Mon Sep 17 00:00:00 2001 From: colorfulberry Date: Tue, 14 Jul 2020 11:34:54 +0800 Subject: [PATCH 03/17] feat: add list and new page --- templates/org/settings/options.tmpl | 182 +++++++++--------- .../repo/cloudbrain/cloudbrain_list.tmpl | 26 +++ templates/repo/cloudbrain/index.tmpl | 35 ++++ templates/repo/cloudbrain/new.tmpl | 30 ++- 4 files changed, 181 insertions(+), 92 deletions(-) create mode 100644 templates/repo/cloudbrain/cloudbrain_list.tmpl diff --git a/templates/org/settings/options.tmpl b/templates/org/settings/options.tmpl index d58c77ad1..228621771 100644 --- a/templates/org/settings/options.tmpl +++ b/templates/org/settings/options.tmpl @@ -1,103 +1,103 @@ {{template "base/head" .}}
- {{template "org/header" .}} -
-
- {{template "org/settings/navbar" .}} -
- {{template "base/alert" .}} -

- {{.i18n.Tr "org.settings.options"}} -

-
-
- {{.CsrfTokenHtml}} -
- - -
-
- - -
-
- - -
-
- - -
-
- - -
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "base/alert" .}} +

+ {{.i18n.Tr "org.settings.options"}} +

+
+ + {{.CsrfTokenHtml}} +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
-
-
- -
-
- - -
-
-
-
- - -
-
-
-
- - -
-
-
+
+
+ +
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
-
- -
-
- - -
-
-
+
+ +
+
+ + +
+
+
- {{if .SignedUser.IsAdmin}} -
+ {{if .SignedUser.IsAdmin}} +
-
- - -

{{.i18n.Tr "admin.users.max_repo_creation_desc"}}

-
- {{end}} +
+ + +

{{.i18n.Tr "admin.users.max_repo_creation_desc"}}

+
+ {{end}} -
- -
- +
+ +
+ -
+
-
- {{.CsrfTokenHtml}} -
- - -
+ + {{.CsrfTokenHtml}} +
+ + +
-
- - {{$.i18n.Tr "settings.delete_current_avatar"}} -
-
-
-
-
-
+
+ + {{$.i18n.Tr "settings.delete_current_avatar"}} +
+ +
+
+
+
{{template "base/footer" .}} diff --git a/templates/repo/cloudbrain/cloudbrain_list.tmpl b/templates/repo/cloudbrain/cloudbrain_list.tmpl new file mode 100644 index 000000000..6a881c779 --- /dev/null +++ b/templates/repo/cloudbrain/cloudbrain_list.tmpl @@ -0,0 +1,26 @@ +
+
+
+ + {{svg "octicon-tasklist" 16}} + 任务名 + +
+
+ waiting +
+
+ {{svg "octicon-flame" 16}} 2020-07-13 16:06:08 +
+ +
+ 18h 0m 8s +
+
+ Stop +
+
+ 再次提交 +
+
+
\ No newline at end of file diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index 399d17520..e0199e9e0 100644 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -1,5 +1,40 @@ {{template "base/head" .}}
{{template "repo/header" .}} +
+
+
+

{{.i18n.Tr "cloudbrain"}}

+
+
+
+
+ {{.i18n.Tr "repo.cloudbrain.new"}} +
+
+
+
+
+
+
+
+
+
+
+
+ + {{.i18n.Tr "repo.issues.filter_sort"}} + +
+
+
+
+
+ {{template "repo/cloudbrain/cloudbrain_list" .}} +
+
+
+
+
{{template "base/footer" .}} diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index 399d17520..394855d9b 100644 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -1,5 +1,33 @@ {{template "base/head" .}} -
+
{{template "repo/header" .}} +
+
+
+ {{.CsrfTokenHtml}} +

+ New Cloudbrain task +

+
+
+
+ + +
+
+ + +
+
+ + + Cancel +
+
+
+
+
{{template "base/footer" .}} From e3e2119a6fdaa506258bc547cc9e6f8963aadc6d Mon Sep 17 00:00:00 2001 From: colorfulberry Date: Tue, 14 Jul 2020 14:32:34 +0800 Subject: [PATCH 04/17] feat: format the tmpl --- templates/repo/cloudbrain/new.tmpl | 58 +++++++++++++++--------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index 394855d9b..00260f6c2 100644 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -1,33 +1,33 @@ {{template "base/head" .}}
- {{template "repo/header" .}} -
-
-
- {{.CsrfTokenHtml}} -

- New Cloudbrain task -

-
-
-
- - -
-
- - -
-
- - - Cancel -
-
-
-
-
+{{template "repo/header" .}} +
+
+
+ {{.CsrfTokenHtml}} +

+ New Cloudbrain task +

+
+
+
+ + +
+
+ + +
+
+ + + Cancel +
+
+
+
+
{{template "base/footer" .}} From ce499a28e3602d80c7dde8b0b4d6e4303d6a086d Mon Sep 17 00:00:00 2001 From: palytoxin Date: Wed, 15 Jul 2020 09:58:41 +0800 Subject: [PATCH 05/17] add repo unit type for cloud brain --- models/repo.go | 6 ++++++ models/repo_unit.go | 20 ++++++++++++++++++-- models/unit.go | 14 ++++++++++++++ modules/auth/repo_form.go | 1 + modules/context/repo.go | 1 + routers/repo/setting.go | 12 ++++++++++++ routers/routes/routes.go | 6 ++++-- templates/repo/header.tmpl | 8 ++++---- templates/repo/settings/options.tmpl | 9 +++++++++ 9 files changed, 69 insertions(+), 8 deletions(-) diff --git a/models/repo.go b/models/repo.go index 1720c0311..1add1dd58 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1076,6 +1076,12 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository) (err error Type: tp, Config: &DatasetConfig{EnableDataset: true}, }) + } else if tp == UnitTypeCloudBrain { + units = append(units, RepoUnit{ + RepoID: repo.ID, + Type: tp, + Config: &CloudBrainConfig{EnableCloudBrain: true}, + }) } else { units = append(units, RepoUnit{ RepoID: repo.ID, diff --git a/models/repo_unit.go b/models/repo_unit.go index ba97c5314..a0e7a2f76 100644 --- a/models/repo_unit.go +++ b/models/repo_unit.go @@ -117,16 +117,30 @@ type DatasetConfig struct { EnableDataset bool } -// FromDB fills up a IssuesConfig from serialized format. +// FromDB fills up a DatasetConfig from serialized format. func (cfg *DatasetConfig) FromDB(bs []byte) error { return json.Unmarshal(bs, &cfg) } -// ToDB exports a IssuesConfig to a serialized format. +// ToDB exports a DatasetConfig to a serialized format. func (cfg *DatasetConfig) ToDB() ([]byte, error) { return json.Marshal(cfg) } +type CloudBrainConfig struct { + EnableCloudBrain bool +} + +// FromDB fills up a CloudBrainConfig from serialized format. +func (cfg *CloudBrainConfig) FromDB(bs []byte) error { + return json.Unmarshal(bs, &cfg) +} + +// ToDB exports a CloudBrainConfig to a serialized format. +func (cfg *CloudBrainConfig) ToDB() ([]byte, error) { + return json.Marshal(cfg) +} + // BeforeSet is invoked from XORM before setting the value of a field of this object. func (r *RepoUnit) BeforeSet(colName string, val xorm.Cell) { switch colName { @@ -144,6 +158,8 @@ func (r *RepoUnit) BeforeSet(colName string, val xorm.Cell) { r.Config = new(IssuesConfig) case UnitTypeDatasets: r.Config = new(DatasetConfig) + case UnitTypeCloudBrain: + r.Config = new(CloudBrainConfig) default: panic("unrecognized repo unit type: " + com.ToStr(*val)) } diff --git a/models/unit.go b/models/unit.go index ef9daf4ba..41712f238 100644 --- a/models/unit.go +++ b/models/unit.go @@ -25,6 +25,7 @@ const ( UnitTypeExternalWiki // 6 ExternalWiki UnitTypeExternalTracker // 7 ExternalTracker UnitTypeDatasets UnitType = 10 // 10 Dataset + UnitTypeCloudBrain UnitType = 11 // 11 CloudBrain ) // Value returns integer value for unit type @@ -50,6 +51,8 @@ func (u UnitType) String() string { return "UnitTypeExternalTracker" case UnitTypeDatasets: return "UnitTypeDataset" + case UnitTypeCloudBrain: + return "UnitTypeCloudBrain" } return fmt.Sprintf("Unknown UnitType %d", u) } @@ -72,6 +75,7 @@ var ( UnitTypeExternalWiki, UnitTypeExternalTracker, UnitTypeDatasets, + UnitTypeCloudBrain, } // DefaultRepoUnits contains the default unit types @@ -82,6 +86,7 @@ var ( UnitTypeReleases, UnitTypeWiki, UnitTypeDatasets, + UnitTypeCloudBrain, } // NotAllowedDefaultRepoUnits contains units that can't be default @@ -255,6 +260,14 @@ var ( 5, } + UnitCloudBrain = Unit{ + UnitTypeCloudBrain, + "repo.cloudbrains", + "/cloudbrains", + "repo.cloudbrains.desc", + 6, + } + // Units contains all the units Units = map[UnitType]Unit{ UnitTypeCode: UnitCode, @@ -265,6 +278,7 @@ var ( UnitTypeWiki: UnitWiki, UnitTypeExternalWiki: UnitExternalWiki, UnitTypeDatasets: UnitDataset, + UnitTypeCloudBrain: UnitCloudBrain, } ) diff --git a/modules/auth/repo_form.go b/modules/auth/repo_form.go index 4f97acb1a..7e867b1af 100644 --- a/modules/auth/repo_form.go +++ b/modules/auth/repo_form.go @@ -121,6 +121,7 @@ type RepoSettingForm struct { // Advanced settings EnableDataset bool + EnableCloudBrain bool EnableWiki bool EnableExternalWiki bool ExternalWikiURL string diff --git a/modules/context/repo.go b/modules/context/repo.go index 9d94030d8..bdd8cde5d 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -814,6 +814,7 @@ func UnitTypes() macaron.Handler { ctx.Data["UnitTypeCode"] = models.UnitTypeCode ctx.Data["UnitTypeIssues"] = models.UnitTypeIssues ctx.Data["UnitTypeDatasets"] = models.UnitTypeDatasets + ctx.Data["UnitTypeCloudBrain"] = models.UnitTypeCloudBrain ctx.Data["UnitTypePullRequests"] = models.UnitTypePullRequests ctx.Data["UnitTypeReleases"] = models.UnitTypeReleases ctx.Data["UnitTypeWiki"] = models.UnitTypeWiki diff --git a/routers/repo/setting.go b/routers/repo/setting.go index dd3490f12..b2ca042de 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -227,6 +227,18 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { deleteUnitTypes = append(deleteUnitTypes, models.UnitTypeDatasets) } + if form.EnableCloudBrain && !models.UnitTypeCloudBrain.UnitGlobalDisabled() { + units = append(units, models.RepoUnit{ + RepoID: repo.ID, + Type: models.UnitTypeCloudBrain, + Config: &models.CloudBrainConfig{ + EnableCloudBrain: form.EnableCloudBrain, + }, + }) + } else if !models.UnitTypeCloudBrain.UnitGlobalDisabled() { + deleteUnitTypes = append(deleteUnitTypes, models.UnitTypeCloudBrain) + } + if form.EnableWiki && form.EnableExternalWiki && !models.UnitTypeExternalWiki.UnitGlobalDisabled() { if !validation.IsValidExternalURL(form.ExternalWikiURL) { ctx.Flash.Error(ctx.Tr("repo.settings.external_wiki_url_error")) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 27c9e0c22..9ffa50059 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -544,6 +544,8 @@ func RegisterRoutes(m *macaron.Macaron) { reqRepoIssuesOrPullsReader := context.RequireRepoReaderOr(models.UnitTypeIssues, models.UnitTypePullRequests) reqRepoDatasetReader := context.RequireRepoReader(models.UnitTypeDatasets) reqRepoDatasetWriter := context.RequireRepoWriter(models.UnitTypeDatasets) + reqRepoCloudBrainReader := context.RequireRepoReader(models.UnitTypeCloudBrain) + reqRepoCloudBrainWriter := context.RequireRepoWriter(models.UnitTypeCloudBrain) // ***** START: Organization ***** m.Group("/org", func() { @@ -870,8 +872,8 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/cloudbrain", func() { - m.Get("", repo.CloudBrainIndex) - m.Get("/new", repo.CloudBrainNew) + m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) + m.Get("/new", reqRepoCloudBrainWriter, repo.CloudBrainNew) }) m.Group("/wiki", func() { diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index b0e546272..7baf67809 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -139,10 +139,10 @@ {{end}} - {{if .Permission.CanRead $.UnitTypeCode}} - - {{svg "octicon-circuit-board" 16}} {{.i18n.Tr "repo.cloudbrain"}} - + {{if .Permission.CanRead $.UnitTypeCloudBrain}} + + {{svg "octicon-file-submodule" 16}} {{.i18n.Tr "cloudbrains"}} + {{end}} {{template "custom/extra_tabs" .}} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 8859b6b2c..e63d369f5 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -150,6 +150,15 @@
+ {{$isCloudBrainEnabled := .Repository.UnitEnabled $.UnitTypeCloudBrain }} +
+ +
+ + +
+
+ {{$isWikiEnabled := or (.Repository.UnitEnabled $.UnitTypeWiki) (.Repository.UnitEnabled $.UnitTypeExternalWiki)}}
From fd8f4255488b818bc1b14ffe126a6583a8949cdf Mon Sep 17 00:00:00 2001 From: palytoxin Date: Sat, 18 Jul 2020 15:36:28 +0800 Subject: [PATCH 06/17] add cloudbrain api support --- go.mod | 3 +- go.sum | 5 + models/cloudbrain.go | 210 ++++++++++++++++++ models/migrations/migrations.go | 1 + models/migrations/v141.go | 26 +++ models/models.go | 1 + modules/auth/cloudbrain.go | 16 ++ modules/cloudbrain/cloudbrain.go | 58 +++++ modules/cloudbrain/resty.go | 120 ++++++++++ modules/setting/cloudbrain.go | 19 ++ routers/repo/cloudbrain.go | 53 ++++- routers/routes/routes.go | 5 +- .../repo/cloudbrain/cloudbrain_list.tmpl | 26 --- templates/repo/cloudbrain/index.tmpl | 32 ++- templates/repo/cloudbrain/new.tmpl | 5 +- vendor/modules.txt | 5 +- 16 files changed, 549 insertions(+), 36 deletions(-) create mode 100644 models/cloudbrain.go create mode 100644 models/migrations/v141.go create mode 100644 modules/auth/cloudbrain.go create mode 100644 modules/cloudbrain/cloudbrain.go create mode 100644 modules/cloudbrain/resty.go create mode 100644 modules/setting/cloudbrain.go delete mode 100644 templates/repo/cloudbrain/cloudbrain_list.tmpl diff --git a/go.mod b/go.mod index 3484f2d76..5f59928c4 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/go-ini/ini v1.56.0 // indirect github.com/go-openapi/jsonreference v0.19.3 // indirect github.com/go-redis/redis v6.15.2+incompatible + github.com/go-resty/resty/v2 v2.3.0 github.com/go-sql-driver/mysql v1.4.1 github.com/go-swagger/go-swagger v0.21.0 github.com/gobwas/glob v0.2.3 @@ -106,7 +107,7 @@ require ( github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60 golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 golang.org/x/mod v0.3.0 // indirect - golang.org/x/net v0.0.0-20200506145744-7e3656a0809f + golang.org/x/net v0.0.0-20200513185701-a91f0712d120 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f golang.org/x/text v0.3.2 diff --git a/go.sum b/go.sum index 0df9a912c..d9d06d409 100644 --- a/go.sum +++ b/go.sum @@ -270,6 +270,9 @@ github.com/go-openapi/validate v0.19.3 h1:PAH/2DylwWcIU1s0Y7k3yNmeAgWOcKrNE2Q7Ww github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-resty/resty v1.12.0 h1:L1P5qymrXL5H/doXe2pKUr1wxovAI5ilm2LdVLbwThc= +github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So= +github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= @@ -740,6 +743,8 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3ob golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f h1:QBjCr1Fz5kw158VqdE9JfI9cJnl/ymnJWAdMuinqL7Y= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/models/cloudbrain.go b/models/cloudbrain.go new file mode 100644 index 000000000..dc97062da --- /dev/null +++ b/models/cloudbrain.go @@ -0,0 +1,210 @@ +package models + +import ( + "fmt" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" +) + +type CloudbrainStatus int8 + +const ( + JobWaiting CloudbrainStatus = iota + JobStopped + JobSucceeded + JobFailed +) + +type Cloudbrain struct { + ID int64 `xorm:"pk autoincr"` + JobID string + // Title string `xorm:"INDEX NOT NULL"` + Status int32 `xorm:"INDEX"` + UserID int64 `xorm:"INDEX"` + RepoID int64 `xorm:"INDEX"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + + User *User `xorm:"-"` + Repo *Repository `xorm:"-"` +} + +type CloudBrainLoginResult struct { + Code string + Msg string + Payload struct { + UserID string `json:"userId"` + RealName string `json:"realName"` + Token string `json:"token"` + Admin bool `json:"admin"` + } +} + +type TaskRole struct { + Name string `json:"name"` + TaskNumber int8 `json:"taskNumber"` + MinSucceededTaskCount int8 `json:"minSucceededTaskCount"` + MinFailedTaskCount int8 `json:"minFailedTaskCount"` + CPUNumber int8 `json:"cpuNumber"` + GPUNumber int8 `json:"gpuNumber"` + MemoryMB int `json:"memoryMB"` + ShmMB int `json:"shmMB"` + Command string `json:"command"` + NeedIBDevice bool `json:"needIBDevice"` + IsMainRole bool `json:"isMainRole"` +} + +type CreateJobParams struct { + JobName string `json:"jobName"` + RetryCount int8 `json:"retryCount"` + GpuType string `json:"gpuType"` + Image string `json:"image"` + TaskRoles []TaskRole `json:"taskRoles"` +} + +type CreateJobResult struct { + Code string + Msg string + Payload struct { + JobID string `json:"jobId"` + } +} + +type GetJobResult struct { + Code string + Msg string + Payload struct { + ID string `json:"Id"` + Name string + Platform string + JobStatus struct { + Username string + State string + SubState string `json:"subState"` + ExecutionType string `json:"executionType"` + Retries int8 `json:"retries"` + CreatedTime int64 `json:"createdTime"` + CompletedTime int64 `json:"completedTime"` + AppID string `json:"appId"` + AppProgress string `json:"appProgress"` + AppTrackingURL string `json:"appTrackingUrl"` + AppLaunchedTime int64 `json:"appLaunchedTime"` + AppCompletedTime int64 `json:"appCompletedTime"` + AppExitCode int8 `json:"appExitCode"` + AppExitDiagnostics string `json:"appExitDiagnostics"` + AppExitType string `json:"appExitType"` + VirtualCluster string `json:"virtualCluster"` + } `json:"jobStatus"` + + TaskRoles string `json:"taskRoles"` + + Resource struct { + CPU int8 `json:"cpu"` + Memory string + GPU string `json:"nvidia.com/gpu"` + } `json:"resource"` + + Config struct { + Image string + JobID string `json:"jobId"` + GpuType string `json:"gpuType"` + JobName string `json:"jobName"` + JobType string `json:"jobType"` + RetryCount int8 `json:"retryCount"` + TaskRoles []struct { + Name string `json:"name"` + ShmMB int32 `json:"shmMB"` + Command string `json:"command"` + MemoryMB int64 `json:"memoryMB"` + CPUNumber int8 `json:"cpuNumber"` + GPUNumber int8 `json:"gpuNumber"` + IsMainRole bool `json:"isMainRole"` + TaskNumber int32 `json:"taskNumber"` + NeedIBDevice bool `json:"needIBDevice"` + MinFailedTaskCount int8 `json:"minFailedTaskCount"` + MinSucceededTaskCount int8 `json:"minSucceededTaskCount"` + } `json:"taskRoles"` + } + + Userinfo struct { + User string + OrgID string `json:"org_id"` + } + } +} + +type CloudbrainsOptions struct { + ListOptions + RepoID int64 // include all repos if empty + UserID int64 + JobStatus CloudbrainStatus + SortType string + CloudbrainIDs []int64 +} + +func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) { + sess := x.NewSession() + defer sess.Close() + + var cond = builder.NewCond() + if opts.RepoID > 0 { + cond.And( + builder.Eq{"cloudbrain.repo_id": opts.RepoID}, + ) + } + + if opts.UserID > 0 { + cond.And( + builder.Eq{"cloudbrain.user_id": opts.UserID}, + ) + } + + switch opts.JobStatus { + case JobWaiting: + cond.And(builder.Eq{"cloudbrain.status": int(JobWaiting)}) + case JobFailed: + cond.And(builder.Eq{"cloudbrain.status": int(JobFailed)}) + case JobStopped: + cond.And(builder.Eq{"cloudbrain.status": int(JobStopped)}) + case JobSucceeded: + cond.And(builder.Eq{"cloudbrain.status": int(JobSucceeded)}) + } + + if len(opts.CloudbrainIDs) > 0 { + cond.And(builder.In("cloudbrain.id", opts.CloudbrainIDs)) + } + + count, err := sess.Where(cond).Count(new(Cloudbrain)) + if err != nil { + return nil, 0, fmt.Errorf("Count: %v", err) + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + sess.Limit(opts.PageSize, start) + } + + sess.OrderBy("cloudbrain.created_unix DESC") + cloudbrains := make([]*Cloudbrain, 0, setting.UI.IssuePagingNum) + if err := sess.Find(&cloudbrains); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + sess.Close() + + return cloudbrains, count, nil +} + +func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { + if _, err = x.Insert(cloudbrain); err != nil { + return err + } + + return nil +} diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 367a8d876..87b79c018 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -213,6 +213,7 @@ var migrations = []Migration{ // v139 -> v140 NewMigration("prepend refs/heads/ to issue refs", prependRefsHeadsToIssueRefs), NewMigration("add dataset migration", addDatasetTable), + NewMigration("add cloudbrain migration", addCloudBrainTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v141.go b/models/migrations/v141.go new file mode 100644 index 000000000..45e9dc2b1 --- /dev/null +++ b/models/migrations/v141.go @@ -0,0 +1,26 @@ +package migrations + +import ( + "fmt" + + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/xorm" +) + +func addCloudBrainTable(x *xorm.Engine) error { + type Cloudbrain struct { + ID int64 `xorm:"pk autoincr"` + JobId string + // Title string `xorm:"INDEX NOT NULL"` + Status int32 `xorm:"INDEX"` + UserID int64 `xorm:"INDEX"` + RepoID int64 `xorm:"INDEX"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + } + + if err := x.Sync2(new(Cloudbrain)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + return nil +} diff --git a/models/models.go b/models/models.go index e3d7d934f..8c73ebba8 100644 --- a/models/models.go +++ b/models/models.go @@ -126,6 +126,7 @@ func init() { new(LanguageStat), new(EmailHash), new(Dataset), + new(Cloudbrain), ) gonicNames := []string{"SSL", "UID"} diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go new file mode 100644 index 000000000..c77204937 --- /dev/null +++ b/modules/auth/cloudbrain.go @@ -0,0 +1,16 @@ +package auth + +import ( + "gitea.com/macaron/binding" + "gitea.com/macaron/macaron" +) + +// CreateDatasetForm form for dataset page +type CreateCloudBrainForm struct { + Image string `binding:"Required"` + Command string `binding:"Required"` +} + +func (f *CreateCloudBrainForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { + return validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go new file mode 100644 index 000000000..a015fa53a --- /dev/null +++ b/modules/cloudbrain/cloudbrain.go @@ -0,0 +1,58 @@ +package cloudbrain + +import ( + "errors" + "fmt" + "strconv" + "time" + + "code.gitea.io/gitea/modules/context" + + "code.gitea.io/gitea/models" +) + +func GenerateTask(ctx *context.Context, image, command string) error { + nowStr := strconv.FormatInt(time.Now().Unix(), 10) + + jobName := fmt.Sprintf("%s%s", ctx.User.Name, nowStr[len(nowStr)-5:]) + jobResult, err := CreateJob(jobName, models.CreateJobParams{ + JobName: jobName, + RetryCount: 1, + GpuType: "debug", + Image: image, + TaskRoles: []models.TaskRole{ + { + Name: jobName, + TaskNumber: 1, + MinSucceededTaskCount: 1, + MinFailedTaskCount: 1, + CPUNumber: 2, + GPUNumber: 1, + MemoryMB: 16384, + ShmMB: 8192, + Command: command, + NeedIBDevice: false, + IsMainRole: false, + }, + }, + }) + if err != nil { + return err + } + if jobResult.Code != "S000" { + return errors.New(jobResult.Msg) + } + + err = models.CreateCloudbrain(&models.Cloudbrain{ + Status: int32(models.JobWaiting), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: jobResult.Payload.JobID, + }) + + if err != nil { + return err + } + + return nil +} diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go new file mode 100644 index 000000000..5f4551a8b --- /dev/null +++ b/modules/cloudbrain/resty.go @@ -0,0 +1,120 @@ +package cloudbrain + +import ( + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + resty "github.com/go-resty/resty/v2" +) + +var ( + restyClient *resty.Client + HOST string + TOKEN string +) + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + } + return restyClient +} + +func checkSetting() { + if len(HOST) != 0 && len(TOKEN) != 0 && restyClient != nil { + return + } + _ = loginCloudbrain() +} + +func loginCloudbrain() error { + conf := setting.GetCloudbrainConfig() + + username := conf.Username + password := conf.Password + HOST = conf.Host + var loginResult models.CloudBrainLoginResult + + client := getRestyClient() + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetBody(map[string]interface{}{"username": username, "password": password, "expiration": "604800"}). + SetResult(&loginResult). + Post(HOST + "/rest-server/api/v1/token") + if err != nil { + return fmt.Errorf("resty loginCloudbrain: %s", err) + } + + if loginResult.Code != "S000" { + return fmt.Errorf("%s: %s", loginResult.Msg, res.String()) + } + + TOKEN = loginResult.Payload.Token + return nil +} + +func CreateJob(jobName string, createJobParams models.CreateJobParams) (*models.CreateJobResult, error) { + checkSetting() + client := getRestyClient() + var jobResult models.CreateJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(createJobParams). + SetResult(&jobResult). + Put(HOST + "/rest-server/api/v1/jobs/" + jobName) + + if err != nil { + return nil, fmt.Errorf("resty create job: %s", err) + } + + if jobResult.Code == "S401" && retry < 1 { + retry++ + _ = loginCloudbrain() + goto sendjob + } + + if jobResult.Code != "S000" { + return &jobResult, fmt.Errorf("jobResult err: %s", res.String()) + } + + return &jobResult, nil +} + +func GetJob(jobID string) (*models.GetJobResult, error) { + checkSetting() + // http://192.168.204.24/rest-server/api/v1/jobs/90e26e500c4b3011ea0a251099a987938b96 + client := getRestyClient() + var getJobResult models.GetJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&getJobResult). + Get(HOST + "/rest-server/api/v1/jobs/" + jobID) + + if err != nil { + return nil, fmt.Errorf("resty GetJob: %s", err) + } + + if getJobResult.Code == "S401" && retry < 1 { + retry++ + _ = loginCloudbrain() + goto sendjob + } + + if getJobResult.Code != "S000" { + return &getJobResult, fmt.Errorf("jobResult GetJob err: %s", res.String()) + } + + return &getJobResult, nil +} diff --git a/modules/setting/cloudbrain.go b/modules/setting/cloudbrain.go new file mode 100644 index 000000000..71d59c697 --- /dev/null +++ b/modules/setting/cloudbrain.go @@ -0,0 +1,19 @@ +package setting + +type CloudbrainLoginConfig struct { + Username string + Password string + Host string +} + +var ( + Cloudbrain = CloudbrainLoginConfig{} +) + +func GetCloudbrainConfig() CloudbrainLoginConfig { + cloudbrainSec := Cfg.Section("cloudbrain") + Cloudbrain.Username = cloudbrainSec.Key("USERNAME").MustString("") + Cloudbrain.Password = cloudbrainSec.Key("PASSWORD").MustString("") + Cloudbrain.Host = cloudbrainSec.Key("HOST").MustString("") + return Cloudbrain +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a6aee1407..d0667e151 100644 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -1,8 +1,12 @@ package repo import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" ) const ( @@ -10,14 +14,59 @@ const ( tplCloudBrainNew base.TplName = "repo/cloudbrain/new" ) +// MustEnableDataset check if repository enable internal cb +func MustEnableCloudbrain(ctx *context.Context) { + if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { + ctx.NotFound("MustEnableCloudbrain", nil) + return + } +} func CloudBrainIndex(ctx *context.Context) { - ctx.Data["PageIsViewCloudBrain"] = true + MustEnableCloudbrain(ctx) + repo := ctx.Repo.Repository + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + + ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repo.ID, + // SortType: sortType, + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return + } + + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) + pager.SetDefaultParams(ctx) + ctx.Data["Page"] = pager + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["Tasks"] = ciTasks ctx.HTML(200, tplCloudBrainIndex) } func CloudBrainNew(ctx *context.Context) { - ctx.Data["PageIsViewCloudBrain"] = true + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["image"] = "192.168.202.74:5000/user-images/deepo:v2.0" + ctx.Data["command"] = `pip3 install jupyterlab==1.1.4;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir=\"/userhome\" --port=80 --NotebookApp.token=\"\" --LabApp.allow_origin=\"self https://cloudbrain.pcl.ac.cn\"` ctx.HTML(200, tplCloudBrainNew) } + +func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { + ctx.Data["PageIsCloudBrain"] = true + image := form.Image + command := form.Command + err := cloudbrain.GenerateTask(ctx, image, command) + if err != nil { + ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) + return + } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 9ffa50059..4f66593de 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -873,8 +873,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/cloudbrain", func() { m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) - m.Get("/new", reqRepoCloudBrainWriter, repo.CloudBrainNew) - }) + m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) + m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) + }, context.RepoRef()) m.Group("/wiki", func() { m.Get("/?:page", repo.Wiki) diff --git a/templates/repo/cloudbrain/cloudbrain_list.tmpl b/templates/repo/cloudbrain/cloudbrain_list.tmpl deleted file mode 100644 index 6a881c779..000000000 --- a/templates/repo/cloudbrain/cloudbrain_list.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -
-
-
- - {{svg "octicon-tasklist" 16}} - 任务名 - -
-
- waiting -
-
- {{svg "octicon-flame" 16}} 2020-07-13 16:06:08 -
- -
- 18h 0m 8s -
-
- Stop -
-
- 再次提交 -
-
-
\ No newline at end of file diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index e0199e9e0..4243d79a9 100644 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -9,7 +9,7 @@
- {{.i18n.Tr "repo.cloudbrain.new"}} + {{.i18n.Tr "repo.cloudbrain.new"}}
@@ -30,7 +30,35 @@
- {{template "repo/cloudbrain/cloudbrain_list" .}} + {{range .Tasks}} +
+
+
+ + {{svg "octicon-tasklist" 16}} + {{.JobID}} + +
+
+ waiting +
+
+ {{svg "octicon-flame" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}} +
+ +
+ 18h 0m 8s +
+
+ {{.Status}} +
+
+ 再次提交 +
+
+
+ {{end}} + {{template "base/paginate" .}}
diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index 00260f6c2..42d12ecf7 100644 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -3,7 +3,8 @@ {{template "repo/header" .}}
-
+ {{template "base/alert" .}} + {{.CsrfTokenHtml}}

New Cloudbrain task @@ -12,7 +13,7 @@
- +
diff --git a/vendor/modules.txt b/vendor/modules.txt index bd8131df5..9828e9e63 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -314,6 +314,9 @@ github.com/go-redis/redis/internal/hashtag github.com/go-redis/redis/internal/pool github.com/go-redis/redis/internal/proto github.com/go-redis/redis/internal/util +# github.com/go-resty/resty/v2 v2.3.0 +## explicit +github.com/go-resty/resty/v2 # github.com/go-sql-driver/mysql v1.4.1 ## explicit github.com/go-sql-driver/mysql @@ -719,7 +722,7 @@ golang.org/x/crypto/ssh/knownhosts ## explicit golang.org/x/mod/module golang.org/x/mod/semver -# golang.org/x/net v0.0.0-20200506145744-7e3656a0809f +# golang.org/x/net v0.0.0-20200513185701-a91f0712d120 ## explicit golang.org/x/net/context golang.org/x/net/context/ctxhttp From 41434423f1c3d68dab057b6831aea320319d82e2 Mon Sep 17 00:00:00 2001 From: palytoxin Date: Mon, 20 Jul 2020 09:42:13 +0800 Subject: [PATCH 07/17] add go-resty mod --- .../github.com/go-resty/resty/v2/.gitignore | 31 + .../github.com/go-resty/resty/v2/.travis.yml | 21 + .../github.com/go-resty/resty/v2/BUILD.bazel | 36 + vendor/github.com/go-resty/resty/v2/LICENSE | 21 + vendor/github.com/go-resty/resty/v2/README.md | 850 +++++++++++++++ vendor/github.com/go-resty/resty/v2/WORKSPACE | 27 + vendor/github.com/go-resty/resty/v2/client.go | 978 ++++++++++++++++++ vendor/github.com/go-resty/resty/v2/go.mod | 5 + .../go-resty/resty/v2/middleware.go | 526 ++++++++++ .../github.com/go-resty/resty/v2/redirect.go | 101 ++ .../github.com/go-resty/resty/v2/request.go | 809 +++++++++++++++ .../github.com/go-resty/resty/v2/response.go | 175 ++++ vendor/github.com/go-resty/resty/v2/resty.go | 40 + vendor/github.com/go-resty/resty/v2/retry.go | 181 ++++ vendor/github.com/go-resty/resty/v2/trace.go | 122 +++ .../github.com/go-resty/resty/v2/transport.go | 35 + .../go-resty/resty/v2/transport112.go | 34 + vendor/github.com/go-resty/resty/v2/util.go | 357 +++++++ 18 files changed, 4349 insertions(+) create mode 100644 vendor/github.com/go-resty/resty/v2/.gitignore create mode 100644 vendor/github.com/go-resty/resty/v2/.travis.yml create mode 100644 vendor/github.com/go-resty/resty/v2/BUILD.bazel create mode 100644 vendor/github.com/go-resty/resty/v2/LICENSE create mode 100644 vendor/github.com/go-resty/resty/v2/README.md create mode 100644 vendor/github.com/go-resty/resty/v2/WORKSPACE create mode 100644 vendor/github.com/go-resty/resty/v2/client.go create mode 100644 vendor/github.com/go-resty/resty/v2/go.mod create mode 100644 vendor/github.com/go-resty/resty/v2/middleware.go create mode 100644 vendor/github.com/go-resty/resty/v2/redirect.go create mode 100644 vendor/github.com/go-resty/resty/v2/request.go create mode 100644 vendor/github.com/go-resty/resty/v2/response.go create mode 100644 vendor/github.com/go-resty/resty/v2/resty.go create mode 100644 vendor/github.com/go-resty/resty/v2/retry.go create mode 100644 vendor/github.com/go-resty/resty/v2/trace.go create mode 100644 vendor/github.com/go-resty/resty/v2/transport.go create mode 100644 vendor/github.com/go-resty/resty/v2/transport112.go create mode 100644 vendor/github.com/go-resty/resty/v2/util.go diff --git a/vendor/github.com/go-resty/resty/v2/.gitignore b/vendor/github.com/go-resty/resty/v2/.gitignore new file mode 100644 index 000000000..54910a04a --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/.gitignore @@ -0,0 +1,31 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +coverage.out +coverage.txt +go.sum + +# Exclude intellij IDE folders +.idea/* \ No newline at end of file diff --git a/vendor/github.com/go-resty/resty/v2/.travis.yml b/vendor/github.com/go-resty/resty/v2/.travis.yml new file mode 100644 index 000000000..fa1ce907d --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/.travis.yml @@ -0,0 +1,21 @@ +language: go + +sudo: false + +go: # use travis ci resource effectively, keep always latest 2 versions and tip :) + - 1.14.x + - 1.13.x + - tip + +install: + - go get -v -t ./... + +script: + - go test ./... -race -coverprofile=coverage.txt -covermode=atomic + +after_success: + - bash <(curl -s https://codecov.io/bash) + +matrix: + allow_failures: + - go: tip diff --git a/vendor/github.com/go-resty/resty/v2/BUILD.bazel b/vendor/github.com/go-resty/resty/v2/BUILD.bazel new file mode 100644 index 000000000..6c47cbbbf --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/BUILD.bazel @@ -0,0 +1,36 @@ +package(default_visibility = ["//visibility:private"]) + +load("@bazel_gazelle//:def.bzl", "gazelle") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +gazelle( + name = "gazelle", + command = "fix", + prefix = "github.com/go-resty/resty/v2", +) + +go_library( + name = "go_default_library", + srcs = glob( + ["*.go"], + exclude = ["*_test.go"], + ), + importpath = "github.com/go-resty/resty/v2", + visibility = ["//visibility:public"], + deps = ["@org_golang_x_net//publicsuffix:go_default_library"], +) + +go_test( + name = "go_default_test", + srcs = + glob( + ["*_test.go"], + exclude = ["example_test.go"], + ), + data = glob([".testdata/*"]), + embed = [":go_default_library"], + importpath = "github.com/go-resty/resty/v2", + deps = [ + "@org_golang_x_net//proxy:go_default_library", + ], +) diff --git a/vendor/github.com/go-resty/resty/v2/LICENSE b/vendor/github.com/go-resty/resty/v2/LICENSE new file mode 100644 index 000000000..c9ae93e39 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-2020 Jeevanandam M., https://myjeeva.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/go-resty/resty/v2/README.md b/vendor/github.com/go-resty/resty/v2/README.md new file mode 100644 index 000000000..4d2199150 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/README.md @@ -0,0 +1,850 @@ +

+

Resty

+

Simple HTTP and REST client library for Go (inspired by Ruby rest-client)

+

Features section describes in detail about Resty capabilities

+

+

+

Build Status Code Coverage Go Report Card Release Version GoDoc License Mentioned in Awesome Go

+

+

+

Resty Communication Channels

+

Chat on Gitter - Resty Community Twitter @go_resty

+

+ +## News + + * v2.3.0 [released](https://github.com/go-resty/resty/releases/tag/v2.3.0) and tagged on May 20, 2020. + * v2.0.0 [released](https://github.com/go-resty/resty/releases/tag/v2.0.0) and tagged on Jul 16, 2019. + * v1.12.0 [released](https://github.com/go-resty/resty/releases/tag/v1.12.0) and tagged on Feb 27, 2019. + * v1.0 released and tagged on Sep 25, 2017. - Resty's first version was released on Sep 15, 2015 then it grew gradually as a very handy and helpful library. Its been a two years since first release. I'm very thankful to Resty users and its [contributors](https://github.com/go-resty/resty/graphs/contributors). + +## Features + + * GET, POST, PUT, DELETE, HEAD, PATCH, OPTIONS, etc. + * Simple and chainable methods for settings and request + * [Request](https://godoc.org/github.com/go-resty/resty#Request) Body can be `string`, `[]byte`, `struct`, `map`, `slice` and `io.Reader` too + * Auto detects `Content-Type` + * Buffer less processing for `io.Reader` + * Request Body can be read multiple times via `Request.RawRequest.GetBody()` + * [Response](https://godoc.org/github.com/go-resty/resty#Response) object gives you more possibility + * Access as `[]byte` array - `response.Body()` OR Access as `string` - `response.String()` + * Know your `response.Time()` and when we `response.ReceivedAt()` + * Automatic marshal and unmarshal for `JSON` and `XML` content type + * Default is `JSON`, if you supply `struct/map` without header `Content-Type` + * For auto-unmarshal, refer to - + - Success scenario [Request.SetResult()](https://godoc.org/github.com/go-resty/resty#Request.SetResult) and [Response.Result()](https://godoc.org/github.com/go-resty/resty#Response.Result). + - Error scenario [Request.SetError()](https://godoc.org/github.com/go-resty/resty#Request.SetError) and [Response.Error()](https://godoc.org/github.com/go-resty/resty#Response.Error). + - Supports [RFC7807](https://tools.ietf.org/html/rfc7807) - `application/problem+json` & `application/problem+xml` + * Easy to upload one or more file(s) via `multipart/form-data` + * Auto detects file content type + * Request URL [Path Params (aka URI Params)](https://godoc.org/github.com/go-resty/resty#Request.SetPathParams) + * Backoff Retry Mechanism with retry condition function [reference](retry_test.go) + * Resty client HTTP & REST [Request](https://godoc.org/github.com/go-resty/resty#Client.OnBeforeRequest) and [Response](https://godoc.org/github.com/go-resty/resty#Client.OnAfterResponse) middlewares + * `Request.SetContext` supported + * Authorization option of `BasicAuth` and `Bearer` token + * Set request `ContentLength` value for all request or particular request + * Custom [Root Certificates](https://godoc.org/github.com/go-resty/resty#Client.SetRootCertificate) and Client [Certificates](https://godoc.org/github.com/go-resty/resty#Client.SetCertificates) + * Download/Save HTTP response directly into File, like `curl -o` flag. See [SetOutputDirectory](https://godoc.org/github.com/go-resty/resty#Client.SetOutputDirectory) & [SetOutput](https://godoc.org/github.com/go-resty/resty#Request.SetOutput). + * Cookies for your request and CookieJar support + * SRV Record based request instead of Host URL + * Client settings like `Timeout`, `RedirectPolicy`, `Proxy`, `TLSClientConfig`, `Transport`, etc. + * Optionally allows GET request with payload, see [SetAllowGetMethodPayload](https://godoc.org/github.com/go-resty/resty#Client.SetAllowGetMethodPayload) + * Supports registering external JSON library into resty, see [how to use](https://github.com/go-resty/resty/issues/76#issuecomment-314015250) + * Exposes Response reader without reading response (no auto-unmarshaling) if need be, see [how to use](https://github.com/go-resty/resty/issues/87#issuecomment-322100604) + * Option to specify expected `Content-Type` when response `Content-Type` header missing. Refer to [#92](https://github.com/go-resty/resty/issues/92) + * Resty design + * Have client level settings & options and also override at Request level if you want to + * Request and Response middlewares + * Create Multiple clients if you want to `resty.New()` + * Supports `http.RoundTripper` implementation, see [SetTransport](https://godoc.org/github.com/go-resty/resty#Client.SetTransport) + * goroutine concurrent safe + * Resty Client trace, see [Client.EnableTrace](https://godoc.org/github.com/go-resty/resty#Client.EnableTrace) and [Request.EnableTrace](https://godoc.org/github.com/go-resty/resty#Request.EnableTrace) + * Debug mode - clean and informative logging presentation + * Gzip - Go does it automatically also resty has fallback handling too + * Works fine with `HTTP/2` and `HTTP/1.1` + * [Bazel support](#bazel-support) + * Easily mock Resty for testing, [for e.g.](#mocking-http-requests-using-httpmock-library) + * Well tested client library + +### Included Batteries + + * Redirect Policies - see [how to use](#redirect-policy) + * NoRedirectPolicy + * FlexibleRedirectPolicy + * DomainCheckRedirectPolicy + * etc. [more info](redirect.go) + * Retry Mechanism [how to use](#retries) + * Backoff Retry + * Conditional Retry + * SRV Record based request instead of Host URL [how to use](resty_test.go#L1412) + * etc (upcoming - throw your idea's [here](https://github.com/go-resty/resty/issues)). + + +#### Supported Go Versions + +Initially Resty started supporting `go modules` since `v1.10.0` release. + +Starting Resty v2 and higher versions, it fully embraces [go modules](https://github.com/golang/go/wiki/Modules) package release. It requires a Go version capable of understanding `/vN` suffixed imports: + +- 1.9.7+ +- 1.10.3+ +- 1.11+ + + +## It might be beneficial for your project :smile: + +Resty author also published following projects for Go Community. + + * [aah framework](https://aahframework.org) - A secure, flexible, rapid Go web framework. + * [THUMBAI](https://thumbai.app) - Go Mod Repository, Go Vanity Service and Simple Proxy Server. + * [go-model](https://github.com/jeevatkm/go-model) - Robust & Easy to use model mapper and utility methods for Go `struct`. + + +## Installation + +```bash +# Go Modules +require github.com/go-resty/resty/v2 v2.3.0 +``` + +## Usage + +The following samples will assist you to become as comfortable as possible with resty library. + +```go +// Import resty into your code and refer it as `resty`. +import "github.com/go-resty/resty/v2" +``` + +#### Simple GET + +```go +// Create a Resty Client +client := resty.New() + +resp, err := client.R(). + EnableTrace(). + Get("https://httpbin.org/get") + +// Explore response object +fmt.Println("Response Info:") +fmt.Println("Error :", err) +fmt.Println("Status Code:", resp.StatusCode()) +fmt.Println("Status :", resp.Status()) +fmt.Println("Proto :", resp.Proto()) +fmt.Println("Time :", resp.Time()) +fmt.Println("Received At:", resp.ReceivedAt()) +fmt.Println("Body :\n", resp) +fmt.Println() + +// Explore trace info +fmt.Println("Request Trace Info:") +ti := resp.Request.TraceInfo() +fmt.Println("DNSLookup :", ti.DNSLookup) +fmt.Println("ConnTime :", ti.ConnTime) +fmt.Println("TCPConnTime :", ti.TCPConnTime) +fmt.Println("TLSHandshake :", ti.TLSHandshake) +fmt.Println("ServerTime :", ti.ServerTime) +fmt.Println("ResponseTime :", ti.ResponseTime) +fmt.Println("TotalTime :", ti.TotalTime) +fmt.Println("IsConnReused :", ti.IsConnReused) +fmt.Println("IsConnWasIdle:", ti.IsConnWasIdle) +fmt.Println("ConnIdleTime :", ti.ConnIdleTime) + +/* Output +Response Info: +Error : +Status Code: 200 +Status : 200 OK +Proto : HTTP/2.0 +Time : 475.611189ms +Received At: 2020-05-19 00:11:06.828188 -0700 PDT m=+0.476510773 +Body : + { + "args": {}, + "headers": { + "Accept-Encoding": "gzip", + "Host": "httpbin.org", + "User-Agent": "go-resty/2.3.0 (https://github.com/go-resty/resty)" + }, + "origin": "0.0.0.0", + "url": "https://httpbin.org/get" +} + +Request Trace Info: +DNSLookup : 4.870246ms +ConnTime : 393.95373ms +TCPConnTime : 78.360432ms +TLSHandshake : 310.032859ms +ServerTime : 81.648284ms +ResponseTime : 124.266µs +TotalTime : 475.611189ms +IsConnReused : false +IsConnWasIdle: false +ConnIdleTime : 0s +*/ +``` + +#### Enhanced GET + +```go +// Create a Resty Client +client := resty.New() + +resp, err := client.R(). + SetQueryParams(map[string]string{ + "page_no": "1", + "limit": "20", + "sort":"name", + "order": "asc", + "random":strconv.FormatInt(time.Now().Unix(), 10), + }). + SetHeader("Accept", "application/json"). + SetAuthToken("BC594900518B4F7EAC75BD37F019E08FBC594900518B4F7EAC75BD37F019E08F"). + Get("/search_result") + + +// Sample of using Request.SetQueryString method +resp, err := client.R(). + SetQueryString("productId=232&template=fresh-sample&cat=resty&source=google&kw=buy a lot more"). + SetHeader("Accept", "application/json"). + SetAuthToken("BC594900518B4F7EAC75BD37F019E08FBC594900518B4F7EAC75BD37F019E08F"). + Get("/show_product") +``` + +#### Various POST method combinations + +```go +// Create a Resty Client +client := resty.New() + +// POST JSON string +// No need to set content type, if you have client level setting +resp, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetBody(`{"username":"testuser", "password":"testpass"}`). + SetResult(&AuthSuccess{}). // or SetResult(AuthSuccess{}). + Post("https://myapp.com/login") + +// POST []byte array +// No need to set content type, if you have client level setting +resp, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetBody([]byte(`{"username":"testuser", "password":"testpass"}`)). + SetResult(&AuthSuccess{}). // or SetResult(AuthSuccess{}). + Post("https://myapp.com/login") + +// POST Struct, default is JSON content type. No need to set one +resp, err := client.R(). + SetBody(User{Username: "testuser", Password: "testpass"}). + SetResult(&AuthSuccess{}). // or SetResult(AuthSuccess{}). + SetError(&AuthError{}). // or SetError(AuthError{}). + Post("https://myapp.com/login") + +// POST Map, default is JSON content type. No need to set one +resp, err := client.R(). + SetBody(map[string]interface{}{"username": "testuser", "password": "testpass"}). + SetResult(&AuthSuccess{}). // or SetResult(AuthSuccess{}). + SetError(&AuthError{}). // or SetError(AuthError{}). + Post("https://myapp.com/login") + +// POST of raw bytes for file upload. For example: upload file to Dropbox +fileBytes, _ := ioutil.ReadFile("/Users/jeeva/mydocument.pdf") + +// See we are not setting content-type header, since go-resty automatically detects Content-Type for you +resp, err := client.R(). + SetBody(fileBytes). + SetContentLength(true). // Dropbox expects this value + SetAuthToken(""). + SetError(&DropboxError{}). // or SetError(DropboxError{}). + Post("https://content.dropboxapi.com/1/files_put/auto/resty/mydocument.pdf") // for upload Dropbox supports PUT too + +// Note: resty detects Content-Type for request body/payload if content type header is not set. +// * For struct and map data type defaults to 'application/json' +// * Fallback is plain text content type +``` + +#### Sample PUT + +You can use various combinations of `PUT` method call like demonstrated for `POST`. + +```go +// Note: This is one sample of PUT method usage, refer POST for more combination + +// Create a Resty Client +client := resty.New() + +// Request goes as JSON content type +// No need to set auth token, error, if you have client level settings +resp, err := client.R(). + SetBody(Article{ + Title: "go-resty", + Content: "This is my article content, oh ya!", + Author: "Jeevanandam M", + Tags: []string{"article", "sample", "resty"}, + }). + SetAuthToken("C6A79608-782F-4ED0-A11D-BD82FAD829CD"). + SetError(&Error{}). // or SetError(Error{}). + Put("https://myapp.com/article/1234") +``` + +#### Sample PATCH + +You can use various combinations of `PATCH` method call like demonstrated for `POST`. + +```go +// Note: This is one sample of PUT method usage, refer POST for more combination + +// Create a Resty Client +client := resty.New() + +// Request goes as JSON content type +// No need to set auth token, error, if you have client level settings +resp, err := client.R(). + SetBody(Article{ + Tags: []string{"new tag1", "new tag2"}, + }). + SetAuthToken("C6A79608-782F-4ED0-A11D-BD82FAD829CD"). + SetError(&Error{}). // or SetError(Error{}). + Patch("https://myapp.com/articles/1234") +``` + +#### Sample DELETE, HEAD, OPTIONS + +```go +// Create a Resty Client +client := resty.New() + +// DELETE a article +// No need to set auth token, error, if you have client level settings +resp, err := client.R(). + SetAuthToken("C6A79608-782F-4ED0-A11D-BD82FAD829CD"). + SetError(&Error{}). // or SetError(Error{}). + Delete("https://myapp.com/articles/1234") + +// DELETE a articles with payload/body as a JSON string +// No need to set auth token, error, if you have client level settings +resp, err := client.R(). + SetAuthToken("C6A79608-782F-4ED0-A11D-BD82FAD829CD"). + SetError(&Error{}). // or SetError(Error{}). + SetHeader("Content-Type", "application/json"). + SetBody(`{article_ids: [1002, 1006, 1007, 87683, 45432] }`). + Delete("https://myapp.com/articles") + +// HEAD of resource +// No need to set auth token, if you have client level settings +resp, err := client.R(). + SetAuthToken("C6A79608-782F-4ED0-A11D-BD82FAD829CD"). + Head("https://myapp.com/videos/hi-res-video") + +// OPTIONS of resource +// No need to set auth token, if you have client level settings +resp, err := client.R(). + SetAuthToken("C6A79608-782F-4ED0-A11D-BD82FAD829CD"). + Options("https://myapp.com/servers/nyc-dc-01") +``` + +### Multipart File(s) upload + +#### Using io.Reader + +```go +profileImgBytes, _ := ioutil.ReadFile("/Users/jeeva/test-img.png") +notesBytes, _ := ioutil.ReadFile("/Users/jeeva/text-file.txt") + +// Create a Resty Client +client := resty.New() + +resp, err := client.R(). + SetFileReader("profile_img", "test-img.png", bytes.NewReader(profileImgBytes)). + SetFileReader("notes", "text-file.txt", bytes.NewReader(notesBytes)). + SetFormData(map[string]string{ + "first_name": "Jeevanandam", + "last_name": "M", + }). + Post("http://myapp.com/upload") +``` + +#### Using File directly from Path + +```go +// Create a Resty Client +client := resty.New() + +// Single file scenario +resp, err := client.R(). + SetFile("profile_img", "/Users/jeeva/test-img.png"). + Post("http://myapp.com/upload") + +// Multiple files scenario +resp, err := client.R(). + SetFiles(map[string]string{ + "profile_img": "/Users/jeeva/test-img.png", + "notes": "/Users/jeeva/text-file.txt", + }). + Post("http://myapp.com/upload") + +// Multipart of form fields and files +resp, err := client.R(). + SetFiles(map[string]string{ + "profile_img": "/Users/jeeva/test-img.png", + "notes": "/Users/jeeva/text-file.txt", + }). + SetFormData(map[string]string{ + "first_name": "Jeevanandam", + "last_name": "M", + "zip_code": "00001", + "city": "my city", + "access_token": "C6A79608-782F-4ED0-A11D-BD82FAD829CD", + }). + Post("http://myapp.com/profile") +``` + +#### Sample Form submission + +```go +// Create a Resty Client +client := resty.New() + +// just mentioning about POST as an example with simple flow +// User Login +resp, err := client.R(). + SetFormData(map[string]string{ + "username": "jeeva", + "password": "mypass", + }). + Post("http://myapp.com/login") + +// Followed by profile update +resp, err := client.R(). + SetFormData(map[string]string{ + "first_name": "Jeevanandam", + "last_name": "M", + "zip_code": "00001", + "city": "new city update", + }). + Post("http://myapp.com/profile") + +// Multi value form data +criteria := url.Values{ + "search_criteria": []string{"book", "glass", "pencil"}, +} +resp, err := client.R(). + SetFormDataFromValues(criteria). + Post("http://myapp.com/search") +``` + +#### Save HTTP Response into File + +```go +// Create a Resty Client +client := resty.New() + +// Setting output directory path, If directory not exists then resty creates one! +// This is optional one, if you're planning using absoule path in +// `Request.SetOutput` and can used together. +client.SetOutputDirectory("/Users/jeeva/Downloads") + +// HTTP response gets saved into file, similar to curl -o flag +_, err := client.R(). + SetOutput("plugin/ReplyWithHeader-v5.1-beta.zip"). + Get("http://bit.ly/1LouEKr") + +// OR using absolute path +// Note: output directory path is not used for absolute path +_, err := client.R(). + SetOutput("/MyDownloads/plugin/ReplyWithHeader-v5.1-beta.zip"). + Get("http://bit.ly/1LouEKr") +``` + +#### Request URL Path Params + +Resty provides easy to use dynamic request URL path params. Params can be set at client and request level. Client level params value can be overridden at request level. + +```go +// Create a Resty Client +client := resty.New() + +client.R().SetPathParams(map[string]string{ + "userId": "sample@sample.com", + "subAccountId": "100002", +}). +Get("/v1/users/{userId}/{subAccountId}/details") + +// Result: +// Composed URL - /v1/users/sample@sample.com/100002/details +``` + +#### Request and Response Middleware + +Resty provides middleware ability to manipulate for Request and Response. It is more flexible than callback approach. + +```go +// Create a Resty Client +client := resty.New() + +// Registering Request Middleware +client.OnBeforeRequest(func(c *resty.Client, req *resty.Request) error { + // Now you have access to Client and current Request object + // manipulate it as per your need + + return nil // if its success otherwise return error + }) + +// Registering Response Middleware +client.OnAfterResponse(func(c *resty.Client, resp *resty.Response) error { + // Now you have access to Client and current Response object + // manipulate it as per your need + + return nil // if its success otherwise return error + }) +``` + +#### Redirect Policy + +Resty provides few ready to use redirect policy(s) also it supports multiple policies together. + +```go +// Create a Resty Client +client := resty.New() + +// Assign Client Redirect Policy. Create one as per you need +client.SetRedirectPolicy(resty.FlexibleRedirectPolicy(15)) + +// Wanna multiple policies such as redirect count, domain name check, etc +client.SetRedirectPolicy(resty.FlexibleRedirectPolicy(20), + resty.DomainCheckRedirectPolicy("host1.com", "host2.org", "host3.net")) +``` + +##### Custom Redirect Policy + +Implement [RedirectPolicy](redirect.go#L20) interface and register it with resty client. Have a look [redirect.go](redirect.go) for more information. + +```go +// Create a Resty Client +client := resty.New() + +// Using raw func into resty.SetRedirectPolicy +client.SetRedirectPolicy(resty.RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { + // Implement your logic here + + // return nil for continue redirect otherwise return error to stop/prevent redirect + return nil +})) + +//--------------------------------------------------- + +// Using struct create more flexible redirect policy +type CustomRedirectPolicy struct { + // variables goes here +} + +func (c *CustomRedirectPolicy) Apply(req *http.Request, via []*http.Request) error { + // Implement your logic here + + // return nil for continue redirect otherwise return error to stop/prevent redirect + return nil +} + +// Registering in resty +client.SetRedirectPolicy(CustomRedirectPolicy{/* initialize variables */}) +``` + +#### Custom Root Certificates and Client Certificates + +```go +// Create a Resty Client +client := resty.New() + +// Custom Root certificates, just supply .pem file. +// you can add one or more root certificates, its get appended +client.SetRootCertificate("/path/to/root/pemFile1.pem") +client.SetRootCertificate("/path/to/root/pemFile2.pem") +// ... and so on! + +// Adding Client Certificates, you add one or more certificates +// Sample for creating certificate object +// Parsing public/private key pair from a pair of files. The files must contain PEM encoded data. +cert1, err := tls.LoadX509KeyPair("certs/client.pem", "certs/client.key") +if err != nil { + log.Fatalf("ERROR client certificate: %s", err) +} +// ... + +// You add one or more certificates +client.SetCertificates(cert1, cert2, cert3) +``` + +#### Custom Root Certificates and Client Certificates from string + +```go +// Custom Root certificates from string +// You can pass you certificates throught env variables as strings +// you can add one or more root certificates, its get appended +client.SetRootCertificateFromString("-----BEGIN CERTIFICATE-----content-----END CERTIFICATE-----") +client.SetRootCertificateFromString("-----BEGIN CERTIFICATE-----content-----END CERTIFICATE-----") +// ... and so on! + +// Adding Client Certificates, you add one or more certificates +// Sample for creating certificate object +// Parsing public/private key pair from a pair of files. The files must contain PEM encoded data. +cert1, err := tls.X509KeyPair([]byte("-----BEGIN CERTIFICATE-----content-----END CERTIFICATE-----"), []byte("-----BEGIN CERTIFICATE-----content-----END CERTIFICATE-----")) +if err != nil { + log.Fatalf("ERROR client certificate: %s", err) +} +// ... + +// You add one or more certificates +client.SetCertificates(cert1, cert2, cert3) +``` + +#### Proxy Settings - Client as well as at Request Level + +Default `Go` supports Proxy via environment variable `HTTP_PROXY`. Resty provides support via `SetProxy` & `RemoveProxy`. +Choose as per your need. + +**Client Level Proxy** settings applied to all the request + +```go +// Create a Resty Client +client := resty.New() + +// Setting a Proxy URL and Port +client.SetProxy("http://proxyserver:8888") + +// Want to remove proxy setting +client.RemoveProxy() +``` + +#### Retries + +Resty uses [backoff](http://www.awsarchitectureblog.com/2015/03/backoff.html) +to increase retry intervals after each attempt. + +Usage example: + +```go +// Create a Resty Client +client := resty.New() + +// Retries are configured per client +client. + // Set retry count to non zero to enable retries + SetRetryCount(3). + // You can override initial retry wait time. + // Default is 100 milliseconds. + SetRetryWaitTime(5 * time.Second). + // MaxWaitTime can be overridden as well. + // Default is 2 seconds. + SetRetryMaxWaitTime(20 * time.Second). + // SetRetryAfter sets callback to calculate wait time between retries. + // Default (nil) implies exponential backoff with jitter + SetRetryAfter(func(client *Client, resp *Response) (time.Duration, error) { + return 0, errors.New("quota exceeded") + }) +``` + +Above setup will result in resty retrying requests returned non nil error up to +3 times with delay increased after each attempt. + +You can optionally provide client with custom retry conditions: + +```go +// Create a Resty Client +client := resty.New() + +client.AddRetryCondition( + // RetryConditionFunc type is for retry condition function + // input: non-nil Response OR request execution error + func(r *resty.Response, err error) bool { + return r.StatusCode() == http.StatusTooManyRequests + }, +) +``` + +Above example will make resty retry requests ended with `429 Too Many Requests` +status code. + +Multiple retry conditions can be added. + +It is also possible to use `resty.Backoff(...)` to get arbitrary retry scenarios +implemented. [Reference](retry_test.go). + +#### Allow GET request with Payload + +```go +// Create a Resty Client +client := resty.New() + +// Allow GET request with Payload. This is disabled by default. +client.SetAllowGetMethodPayload(true) +``` + +#### Wanna Multiple Clients + +```go +// Here you go! +// Client 1 +client1 := resty.New() +client1.R().Get("http://httpbin.org") +// ... + +// Client 2 +client2 := resty.New() +client2.R().Head("http://httpbin.org") +// ... + +// Bend it as per your need!!! +``` + +#### Remaining Client Settings & its Options + +```go +// Create a Resty Client +client := resty.New() + +// Unique settings at Client level +//-------------------------------- +// Enable debug mode +client.SetDebug(true) + +// Assign Client TLSClientConfig +// One can set custom root-certificate. Refer: http://golang.org/pkg/crypto/tls/#example_Dial +client.SetTLSClientConfig(&tls.Config{ RootCAs: roots }) + +// or One can disable security check (https) +client.SetTLSClientConfig(&tls.Config{ InsecureSkipVerify: true }) + +// Set client timeout as per your need +client.SetTimeout(1 * time.Minute) + + +// You can override all below settings and options at request level if you want to +//-------------------------------------------------------------------------------- +// Host URL for all request. So you can use relative URL in the request +client.SetHostURL("http://httpbin.org") + +// Headers for all request +client.SetHeader("Accept", "application/json") +client.SetHeaders(map[string]string{ + "Content-Type": "application/json", + "User-Agent": "My custom User Agent String", + }) + +// Cookies for all request +client.SetCookie(&http.Cookie{ + Name:"go-resty", + Value:"This is cookie value", + Path: "/", + Domain: "sample.com", + MaxAge: 36000, + HttpOnly: true, + Secure: false, + }) +client.SetCookies(cookies) + +// URL query parameters for all request +client.SetQueryParam("user_id", "00001") +client.SetQueryParams(map[string]string{ // sample of those who use this manner + "api_key": "api-key-here", + "api_secert": "api-secert", + }) +client.R().SetQueryString("productId=232&template=fresh-sample&cat=resty&source=google&kw=buy a lot more") + +// Form data for all request. Typically used with POST and PUT +client.SetFormData(map[string]string{ + "access_token": "BC594900-518B-4F7E-AC75-BD37F019E08F", + }) + +// Basic Auth for all request +client.SetBasicAuth("myuser", "mypass") + +// Bearer Auth Token for all request +client.SetAuthToken("BC594900518B4F7EAC75BD37F019E08FBC594900518B4F7EAC75BD37F019E08F") + +// Enabling Content length value for all request +client.SetContentLength(true) + +// Registering global Error object structure for JSON/XML request +client.SetError(&Error{}) // or resty.SetError(Error{}) +``` + +#### Unix Socket + +```go +unixSocket := "/var/run/my_socket.sock" + +// Create a Go's http.Transport so we can set it in resty. +transport := http.Transport{ + Dial: func(_, _ string) (net.Conn, error) { + return net.Dial("unix", unixSocket) + }, +} + +// Create a Resty Client +client := resty.New() + +// Set the previous transport that we created, set the scheme of the communication to the +// socket and set the unixSocket as the HostURL. +client.SetTransport(&transport).SetScheme("http").SetHostURL(unixSocket) + +// No need to write the host's URL on the request, just the path. +client.R().Get("/index.html") +``` + +#### Bazel support + +Resty can be built, tested and depended upon via [Bazel](https://bazel.build). +For example, to run all tests: + +```shell +bazel test :go_default_test +``` + +#### Mocking http requests using [httpmock](https://github.com/jarcoal/httpmock) library + +In order to mock the http requests when testing your application you +could use the `httpmock` library. + +When using the default resty client, you should pass the client to the library as follow: + +```go +// Create a Resty Client +client := resty.New() + +// Get the underlying HTTP Client and set it to Mock +httpmock.ActivateNonDefault(client.GetClient()) +``` + +More detailed example of mocking resty http requests using ginko could be found [here](https://github.com/jarcoal/httpmock#ginkgo--resty-example). + +## Versioning + +Resty releases versions according to [Semantic Versioning](http://semver.org) + + * Resty v2 does not use `gopkg.in` service for library versioning. + * Resty fully adapted to `go mod` capabilities since `v1.10.0` release. + * Resty v1 series was using `gopkg.in` to provide versioning. `gopkg.in/resty.vX` points to appropriate tagged versions; `X` denotes version series number and it's a stable release for production use. For e.g. `gopkg.in/resty.v0`. + * Development takes place at the master branch. Although the code in master should always compile and test successfully, it might break API's. I aim to maintain backwards compatibility, but sometimes API's and behavior might be changed to fix a bug. + +## Contribution + +I would welcome your contribution! If you find any improvement or issue you want to fix, feel free to send a pull request, I like pull requests that include test cases for fix/enhancement. I have done my best to bring pretty good code coverage. Feel free to write tests. + +BTW, I'd like to know what you think about `Resty`. Kindly open an issue or send me an email; it'd mean a lot to me. + +## Creator + +[Jeevanandam M.](https://github.com/jeevatkm) (jeeva@myjeeva.com) + +## Core Team + +Have a look on [Members](https://github.com/orgs/go-resty/teams/core/members) page. + +## Contributors + +Have a look on [Contributors](https://github.com/go-resty/resty/graphs/contributors) page. + +## License + +Resty released under MIT license, refer [LICENSE](LICENSE) file. diff --git a/vendor/github.com/go-resty/resty/v2/WORKSPACE b/vendor/github.com/go-resty/resty/v2/WORKSPACE new file mode 100644 index 000000000..5459d6321 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/WORKSPACE @@ -0,0 +1,27 @@ +workspace(name = "resty") + +git_repository( + name = "io_bazel_rules_go", + remote = "https://github.com/bazelbuild/rules_go.git", + tag = "0.13.0", +) + +git_repository( + name = "bazel_gazelle", + remote = "https://github.com/bazelbuild/bazel-gazelle.git", + tag = "0.13.0", +) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_rules_dependencies", + "go_register_toolchains", +) + +go_rules_dependencies() + +go_register_toolchains() + +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") + +gazelle_dependencies() diff --git a/vendor/github.com/go-resty/resty/v2/client.go b/vendor/github.com/go-resty/resty/v2/client.go new file mode 100644 index 000000000..7b51657d2 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/client.go @@ -0,0 +1,978 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "bytes" + "compress/gzip" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "math" + "net/http" + "net/url" + "reflect" + "regexp" + "strings" + "sync" + "time" +) + +const ( + // MethodGet HTTP method + MethodGet = "GET" + + // MethodPost HTTP method + MethodPost = "POST" + + // MethodPut HTTP method + MethodPut = "PUT" + + // MethodDelete HTTP method + MethodDelete = "DELETE" + + // MethodPatch HTTP method + MethodPatch = "PATCH" + + // MethodHead HTTP method + MethodHead = "HEAD" + + // MethodOptions HTTP method + MethodOptions = "OPTIONS" +) + +var ( + hdrUserAgentKey = http.CanonicalHeaderKey("User-Agent") + hdrAcceptKey = http.CanonicalHeaderKey("Accept") + hdrContentTypeKey = http.CanonicalHeaderKey("Content-Type") + hdrContentLengthKey = http.CanonicalHeaderKey("Content-Length") + hdrContentEncodingKey = http.CanonicalHeaderKey("Content-Encoding") + hdrAuthorizationKey = http.CanonicalHeaderKey("Authorization") + + plainTextType = "text/plain; charset=utf-8" + jsonContentType = "application/json" + formContentType = "application/x-www-form-urlencoded" + + jsonCheck = regexp.MustCompile(`(?i:(application|text)/(json|.*\+json|json\-.*)(;|$))`) + xmlCheck = regexp.MustCompile(`(?i:(application|text)/(xml|.*\+xml)(;|$))`) + + hdrUserAgentValue = "go-resty/" + Version + " (https://github.com/go-resty/resty)" + bufPool = &sync.Pool{New: func() interface{} { return &bytes.Buffer{} }} +) + +type ( + // RequestMiddleware type is for request middleware, called before a request is sent + RequestMiddleware func(*Client, *Request) error + + // ResponseMiddleware type is for response middleware, called after a response has been received + ResponseMiddleware func(*Client, *Response) error + + // PreRequestHook type is for the request hook, called right before the request is sent + PreRequestHook func(*Client, *http.Request) error + + // RequestLogCallback type is for request logs, called before the request is logged + RequestLogCallback func(*RequestLog) error + + // ResponseLogCallback type is for response logs, called before the response is logged + ResponseLogCallback func(*ResponseLog) error +) + +// Client struct is used to create Resty client with client level settings, +// these settings are applicable to all the request raised from the client. +// +// Resty also provides an options to override most of the client settings +// at request level. +type Client struct { + HostURL string + QueryParam url.Values + FormData url.Values + Header http.Header + UserInfo *User + Token string + AuthScheme string + Cookies []*http.Cookie + Error reflect.Type + Debug bool + DisableWarn bool + AllowGetMethodPayload bool + RetryCount int + RetryWaitTime time.Duration + RetryMaxWaitTime time.Duration + RetryConditions []RetryConditionFunc + RetryAfter RetryAfterFunc + JSONMarshal func(v interface{}) ([]byte, error) + JSONUnmarshal func(data []byte, v interface{}) error + + jsonEscapeHTML bool + setContentLength bool + closeConnection bool + notParseResponse bool + trace bool + debugBodySizeLimit int64 + outputDirectory string + scheme string + pathParams map[string]string + log Logger + httpClient *http.Client + proxyURL *url.URL + beforeRequest []RequestMiddleware + udBeforeRequest []RequestMiddleware + preReqHook PreRequestHook + afterResponse []ResponseMiddleware + requestLog RequestLogCallback + responseLog ResponseLogCallback +} + +// User type is to hold an username and password information +type User struct { + Username, Password string +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Client methods +//___________________________________ + +// SetHostURL method is to set Host URL in the client instance. It will be used with request +// raised from this client with relative URL +// // Setting HTTP address +// client.SetHostURL("http://myjeeva.com") +// +// // Setting HTTPS address +// client.SetHostURL("https://myjeeva.com") +func (c *Client) SetHostURL(url string) *Client { + c.HostURL = strings.TrimRight(url, "/") + return c +} + +// SetHeader method sets a single header field and its value in the client instance. +// These headers will be applied to all requests raised from this client instance. +// Also it can be overridden at request level header options. +// +// See `Request.SetHeader` or `Request.SetHeaders`. +// +// For Example: To set `Content-Type` and `Accept` as `application/json` +// +// client. +// SetHeader("Content-Type", "application/json"). +// SetHeader("Accept", "application/json") +func (c *Client) SetHeader(header, value string) *Client { + c.Header.Set(header, value) + return c +} + +// SetHeaders method sets multiple headers field and its values at one go in the client instance. +// These headers will be applied to all requests raised from this client instance. Also it can be +// overridden at request level headers options. +// +// See `Request.SetHeaders` or `Request.SetHeader`. +// +// For Example: To set `Content-Type` and `Accept` as `application/json` +// +// client.SetHeaders(map[string]string{ +// "Content-Type": "application/json", +// "Accept": "application/json", +// }) +func (c *Client) SetHeaders(headers map[string]string) *Client { + for h, v := range headers { + c.Header.Set(h, v) + } + return c +} + +// SetCookieJar method sets custom http.CookieJar in the resty client. Its way to override default. +// +// For Example: sometimes we don't want to save cookies in api contacting, we can remove the default +// CookieJar in resty client. +// +// client.SetCookieJar(nil) +func (c *Client) SetCookieJar(jar http.CookieJar) *Client { + c.httpClient.Jar = jar + return c +} + +// SetCookie method appends a single cookie in the client instance. +// These cookies will be added to all the request raised from this client instance. +// client.SetCookie(&http.Cookie{ +// Name:"go-resty", +// Value:"This is cookie value", +// }) +func (c *Client) SetCookie(hc *http.Cookie) *Client { + c.Cookies = append(c.Cookies, hc) + return c +} + +// SetCookies method sets an array of cookies in the client instance. +// These cookies will be added to all the request raised from this client instance. +// cookies := []*http.Cookie{ +// &http.Cookie{ +// Name:"go-resty-1", +// Value:"This is cookie 1 value", +// }, +// &http.Cookie{ +// Name:"go-resty-2", +// Value:"This is cookie 2 value", +// }, +// } +// +// // Setting a cookies into resty +// client.SetCookies(cookies) +func (c *Client) SetCookies(cs []*http.Cookie) *Client { + c.Cookies = append(c.Cookies, cs...) + return c +} + +// SetQueryParam method sets single parameter and its value in the client instance. +// It will be formed as query string for the request. +// +// For Example: `search=kitchen%20papers&size=large` +// in the URL after `?` mark. These query params will be added to all the request raised from +// this client instance. Also it can be overridden at request level Query Param options. +// +// See `Request.SetQueryParam` or `Request.SetQueryParams`. +// client. +// SetQueryParam("search", "kitchen papers"). +// SetQueryParam("size", "large") +func (c *Client) SetQueryParam(param, value string) *Client { + c.QueryParam.Set(param, value) + return c +} + +// SetQueryParams method sets multiple parameters and their values at one go in the client instance. +// It will be formed as query string for the request. +// +// For Example: `search=kitchen%20papers&size=large` +// in the URL after `?` mark. These query params will be added to all the request raised from this +// client instance. Also it can be overridden at request level Query Param options. +// +// See `Request.SetQueryParams` or `Request.SetQueryParam`. +// client.SetQueryParams(map[string]string{ +// "search": "kitchen papers", +// "size": "large", +// }) +func (c *Client) SetQueryParams(params map[string]string) *Client { + for p, v := range params { + c.SetQueryParam(p, v) + } + return c +} + +// SetFormData method sets Form parameters and their values in the client instance. +// It's applicable only HTTP method `POST` and `PUT` and requets content type would be set as +// `application/x-www-form-urlencoded`. These form data will be added to all the request raised from +// this client instance. Also it can be overridden at request level form data. +// +// See `Request.SetFormData`. +// client.SetFormData(map[string]string{ +// "access_token": "BC594900-518B-4F7E-AC75-BD37F019E08F", +// "user_id": "3455454545", +// }) +func (c *Client) SetFormData(data map[string]string) *Client { + for k, v := range data { + c.FormData.Set(k, v) + } + return c +} + +// SetBasicAuth method sets the basic authentication header in the HTTP request. For Example: +// Authorization: Basic +// +// For Example: To set the header for username "go-resty" and password "welcome" +// client.SetBasicAuth("go-resty", "welcome") +// +// This basic auth information gets added to all the request rasied from this client instance. +// Also it can be overridden or set one at the request level is supported. +// +// See `Request.SetBasicAuth`. +func (c *Client) SetBasicAuth(username, password string) *Client { + c.UserInfo = &User{Username: username, Password: password} + return c +} + +// SetAuthToken method sets the auth token of the `Authorization` header for all HTTP requests. +// The default auth scheme is `Bearer`, it can be customized with the method `SetAuthScheme`. For Example: +// Authorization: +// +// For Example: To set auth token BC594900518B4F7EAC75BD37F019E08FBC594900518B4F7EAC75BD37F019E08F +// +// client.SetAuthToken("BC594900518B4F7EAC75BD37F019E08FBC594900518B4F7EAC75BD37F019E08F") +// +// This auth token gets added to all the requests rasied from this client instance. +// Also it can be overridden or set one at the request level is supported. +// +// See `Request.SetAuthToken`. +func (c *Client) SetAuthToken(token string) *Client { + c.Token = token + return c +} + +// SetAuthScheme method sets the auth scheme type in the HTTP request. For Example: +// Authorization: +// +// For Example: To set the scheme to use OAuth +// +// client.SetAuthScheme("OAuth") +// +// This auth scheme gets added to all the requests rasied from this client instance. +// Also it can be overridden or set one at the request level is supported. +// +// Information about auth schemes can be found in RFC7235 which is linked to below +// along with the page containing the currently defined official authentication schemes: +// https://tools.ietf.org/html/rfc7235 +// https://www.iana.org/assignments/http-authschemes/http-authschemes.xhtml#authschemes +// +// See `Request.SetAuthToken`. +func (c *Client) SetAuthScheme(scheme string) *Client { + c.AuthScheme = scheme + return c +} + +// R method creates a new request instance, its used for Get, Post, Put, Delete, Patch, Head, Options, etc. +func (c *Client) R() *Request { + r := &Request{ + QueryParam: url.Values{}, + FormData: url.Values{}, + Header: http.Header{}, + Cookies: make([]*http.Cookie, 0), + + client: c, + multipartFiles: []*File{}, + multipartFields: []*MultipartField{}, + pathParams: map[string]string{}, + jsonEscapeHTML: true, + } + return r +} + +// NewRequest is an alias for method `R()`. Creates a new request instance, its used for +// Get, Post, Put, Delete, Patch, Head, Options, etc. +func (c *Client) NewRequest() *Request { + return c.R() +} + +// OnBeforeRequest method appends request middleware into the before request chain. +// Its gets applied after default Resty request middlewares and before request +// been sent from Resty to host server. +// client.OnBeforeRequest(func(c *resty.Client, r *resty.Request) error { +// // Now you have access to Client and Request instance +// // manipulate it as per your need +// +// return nil // if its success otherwise return error +// }) +func (c *Client) OnBeforeRequest(m RequestMiddleware) *Client { + c.udBeforeRequest = append(c.udBeforeRequest, m) + return c +} + +// OnAfterResponse method appends response middleware into the after response chain. +// Once we receive response from host server, default Resty response middleware +// gets applied and then user assigened response middlewares applied. +// client.OnAfterResponse(func(c *resty.Client, r *resty.Response) error { +// // Now you have access to Client and Response instance +// // manipulate it as per your need +// +// return nil // if its success otherwise return error +// }) +func (c *Client) OnAfterResponse(m ResponseMiddleware) *Client { + c.afterResponse = append(c.afterResponse, m) + return c +} + +// SetPreRequestHook method sets the given pre-request function into resty client. +// It is called right before the request is fired. +// +// Note: Only one pre-request hook can be registered. Use `client.OnBeforeRequest` for mutilple. +func (c *Client) SetPreRequestHook(h PreRequestHook) *Client { + if c.preReqHook != nil { + c.log.Warnf("Overwriting an existing pre-request hook: %s", functionName(h)) + } + c.preReqHook = h + return c +} + +// SetDebug method enables the debug mode on Resty client. Client logs details of every request and response. +// For `Request` it logs information such as HTTP verb, Relative URL path, Host, Headers, Body if it has one. +// For `Response` it logs information such as Status, Response Time, Headers, Body if it has one. +// client.SetDebug(true) +func (c *Client) SetDebug(d bool) *Client { + c.Debug = d + return c +} + +// SetDebugBodyLimit sets the maximum size for which the response and request body will be logged in debug mode. +// client.SetDebugBodyLimit(1000000) +func (c *Client) SetDebugBodyLimit(sl int64) *Client { + c.debugBodySizeLimit = sl + return c +} + +// OnRequestLog method used to set request log callback into Resty. Registered callback gets +// called before the resty actually logs the information. +func (c *Client) OnRequestLog(rl RequestLogCallback) *Client { + if c.requestLog != nil { + c.log.Warnf("Overwriting an existing on-request-log callback from=%s to=%s", + functionName(c.requestLog), functionName(rl)) + } + c.requestLog = rl + return c +} + +// OnResponseLog method used to set response log callback into Resty. Registered callback gets +// called before the resty actually logs the information. +func (c *Client) OnResponseLog(rl ResponseLogCallback) *Client { + if c.responseLog != nil { + c.log.Warnf("Overwriting an existing on-response-log callback from=%s to=%s", + functionName(c.responseLog), functionName(rl)) + } + c.responseLog = rl + return c +} + +// SetDisableWarn method disables the warning message on Resty client. +// +// For Example: Resty warns the user when BasicAuth used on non-TLS mode. +// client.SetDisableWarn(true) +func (c *Client) SetDisableWarn(d bool) *Client { + c.DisableWarn = d + return c +} + +// SetAllowGetMethodPayload method allows the GET method with payload on Resty client. +// +// For Example: Resty allows the user sends request with a payload on HTTP GET method. +// client.SetAllowGetMethodPayload(true) +func (c *Client) SetAllowGetMethodPayload(a bool) *Client { + c.AllowGetMethodPayload = a + return c +} + +// SetLogger method sets given writer for logging Resty request and response details. +// +// Compliant to interface `resty.Logger`. +func (c *Client) SetLogger(l Logger) *Client { + c.log = l + return c +} + +// SetContentLength method enables the HTTP header `Content-Length` value for every request. +// By default Resty won't set `Content-Length`. +// client.SetContentLength(true) +// +// Also you have an option to enable for particular request. See `Request.SetContentLength` +func (c *Client) SetContentLength(l bool) *Client { + c.setContentLength = l + return c +} + +// SetTimeout method sets timeout for request raised from client. +// client.SetTimeout(time.Duration(1 * time.Minute)) +func (c *Client) SetTimeout(timeout time.Duration) *Client { + c.httpClient.Timeout = timeout + return c +} + +// SetError method is to register the global or client common `Error` object into Resty. +// It is used for automatic unmarshalling if response status code is greater than 399 and +// content type either JSON or XML. Can be pointer or non-pointer. +// client.SetError(&Error{}) +// // OR +// client.SetError(Error{}) +func (c *Client) SetError(err interface{}) *Client { + c.Error = typeOf(err) + return c +} + +// SetRedirectPolicy method sets the client redirect poilicy. Resty provides ready to use +// redirect policies. Wanna create one for yourself refer to `redirect.go`. +// +// client.SetRedirectPolicy(FlexibleRedirectPolicy(20)) +// +// // Need multiple redirect policies together +// client.SetRedirectPolicy(FlexibleRedirectPolicy(20), DomainCheckRedirectPolicy("host1.com", "host2.net")) +func (c *Client) SetRedirectPolicy(policies ...interface{}) *Client { + for _, p := range policies { + if _, ok := p.(RedirectPolicy); !ok { + c.log.Errorf("%v does not implement resty.RedirectPolicy (missing Apply method)", + functionName(p)) + } + } + + c.httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + for _, p := range policies { + if err := p.(RedirectPolicy).Apply(req, via); err != nil { + return err + } + } + return nil // looks good, go ahead + } + + return c +} + +// SetRetryCount method enables retry on Resty client and allows you +// to set no. of retry count. Resty uses a Backoff mechanism. +func (c *Client) SetRetryCount(count int) *Client { + c.RetryCount = count + return c +} + +// SetRetryWaitTime method sets default wait time to sleep before retrying +// request. +// +// Default is 100 milliseconds. +func (c *Client) SetRetryWaitTime(waitTime time.Duration) *Client { + c.RetryWaitTime = waitTime + return c +} + +// SetRetryMaxWaitTime method sets max wait time to sleep before retrying +// request. +// +// Default is 2 seconds. +func (c *Client) SetRetryMaxWaitTime(maxWaitTime time.Duration) *Client { + c.RetryMaxWaitTime = maxWaitTime + return c +} + +// SetRetryAfter sets callback to calculate wait time between retries. +// Default (nil) implies exponential backoff with jitter +func (c *Client) SetRetryAfter(callback RetryAfterFunc) *Client { + c.RetryAfter = callback + return c +} + +// AddRetryCondition method adds a retry condition function to array of functions +// that are checked to determine if the request is retried. The request will +// retry if any of the functions return true and error is nil. +func (c *Client) AddRetryCondition(condition RetryConditionFunc) *Client { + c.RetryConditions = append(c.RetryConditions, condition) + return c +} + +// SetTLSClientConfig method sets TLSClientConfig for underling client Transport. +// +// For Example: +// // One can set custom root-certificate. Refer: http://golang.org/pkg/crypto/tls/#example_Dial +// client.SetTLSClientConfig(&tls.Config{ RootCAs: roots }) +// +// // or One can disable security check (https) +// client.SetTLSClientConfig(&tls.Config{ InsecureSkipVerify: true }) +// +// Note: This method overwrites existing `TLSClientConfig`. +func (c *Client) SetTLSClientConfig(config *tls.Config) *Client { + transport, err := c.transport() + if err != nil { + c.log.Errorf("%v", err) + return c + } + transport.TLSClientConfig = config + return c +} + +// SetProxy method sets the Proxy URL and Port for Resty client. +// client.SetProxy("http://proxyserver:8888") +// +// OR Without this `SetProxy` method, you could also set Proxy via environment variable. +// +// Refer to godoc `http.ProxyFromEnvironment`. +func (c *Client) SetProxy(proxyURL string) *Client { + transport, err := c.transport() + if err != nil { + c.log.Errorf("%v", err) + return c + } + + pURL, err := url.Parse(proxyURL) + if err != nil { + c.log.Errorf("%v", err) + return c + } + + c.proxyURL = pURL + transport.Proxy = http.ProxyURL(c.proxyURL) + return c +} + +// RemoveProxy method removes the proxy configuration from Resty client +// client.RemoveProxy() +func (c *Client) RemoveProxy() *Client { + transport, err := c.transport() + if err != nil { + c.log.Errorf("%v", err) + return c + } + c.proxyURL = nil + transport.Proxy = nil + return c +} + +// SetCertificates method helps to set client certificates into Resty conveniently. +func (c *Client) SetCertificates(certs ...tls.Certificate) *Client { + config, err := c.tlsConfig() + if err != nil { + c.log.Errorf("%v", err) + return c + } + config.Certificates = append(config.Certificates, certs...) + return c +} + +// SetRootCertificate method helps to add one or more root certificates into Resty client +// client.SetRootCertificate("/path/to/root/pemFile.pem") +func (c *Client) SetRootCertificate(pemFilePath string) *Client { + rootPemData, err := ioutil.ReadFile(pemFilePath) + if err != nil { + c.log.Errorf("%v", err) + return c + } + + config, err := c.tlsConfig() + if err != nil { + c.log.Errorf("%v", err) + return c + } + if config.RootCAs == nil { + config.RootCAs = x509.NewCertPool() + } + + config.RootCAs.AppendCertsFromPEM(rootPemData) + return c +} + +// SetRootCertificateFromString method helps to add one or more root certificates into Resty client +// client.SetRootCertificateFromString("pem file content") +func (c *Client) SetRootCertificateFromString(pemContent string) *Client { + config, err := c.tlsConfig() + if err != nil { + c.log.Errorf("%v", err) + return c + } + if config.RootCAs == nil { + config.RootCAs = x509.NewCertPool() + } + + config.RootCAs.AppendCertsFromPEM([]byte(pemContent)) + return c +} + +// SetOutputDirectory method sets output directory for saving HTTP response into file. +// If the output directory not exists then resty creates one. This setting is optional one, +// if you're planning using absolute path in `Request.SetOutput` and can used together. +// client.SetOutputDirectory("/save/http/response/here") +func (c *Client) SetOutputDirectory(dirPath string) *Client { + c.outputDirectory = dirPath + return c +} + +// SetTransport method sets custom `*http.Transport` or any `http.RoundTripper` +// compatible interface implementation in the resty client. +// +// Note: +// +// - If transport is not type of `*http.Transport` then you may not be able to +// take advantage of some of the Resty client settings. +// +// - It overwrites the Resty client transport instance and it's configurations. +// +// transport := &http.Transport{ +// // somthing like Proxying to httptest.Server, etc... +// Proxy: func(req *http.Request) (*url.URL, error) { +// return url.Parse(server.URL) +// }, +// } +// +// client.SetTransport(transport) +func (c *Client) SetTransport(transport http.RoundTripper) *Client { + if transport != nil { + c.httpClient.Transport = transport + } + return c +} + +// SetScheme method sets custom scheme in the Resty client. It's way to override default. +// client.SetScheme("http") +func (c *Client) SetScheme(scheme string) *Client { + if !IsStringEmpty(scheme) { + c.scheme = scheme + } + return c +} + +// SetCloseConnection method sets variable `Close` in http request struct with the given +// value. More info: https://golang.org/src/net/http/request.go +func (c *Client) SetCloseConnection(close bool) *Client { + c.closeConnection = close + return c +} + +// SetDoNotParseResponse method instructs `Resty` not to parse the response body automatically. +// Resty exposes the raw response body as `io.ReadCloser`. Also do not forget to close the body, +// otherwise you might get into connection leaks, no connection reuse. +// +// Note: Response middlewares are not applicable, if you use this option. Basically you have +// taken over the control of response parsing from `Resty`. +func (c *Client) SetDoNotParseResponse(parse bool) *Client { + c.notParseResponse = parse + return c +} + +// SetPathParams method sets multiple URL path key-value pairs at one go in the +// Resty client instance. +// client.SetPathParams(map[string]string{ +// "userId": "sample@sample.com", +// "subAccountId": "100002", +// }) +// +// Result: +// URL - /v1/users/{userId}/{subAccountId}/details +// Composed URL - /v1/users/sample@sample.com/100002/details +// It replace the value of the key while composing request URL. Also it can be +// overridden at request level Path Params options, see `Request.SetPathParams`. +func (c *Client) SetPathParams(params map[string]string) *Client { + for p, v := range params { + c.pathParams[p] = v + } + return c +} + +// SetJSONEscapeHTML method is to enable/disable the HTML escape on JSON marshal. +// +// Note: This option only applicable to standard JSON Marshaller. +func (c *Client) SetJSONEscapeHTML(b bool) *Client { + c.jsonEscapeHTML = b + return c +} + +// EnableTrace method enables the Resty client trace for the requests fired from +// the client using `httptrace.ClientTrace` and provides insights. +// +// client := resty.New().EnableTrace() +// +// resp, err := client.R().Get("https://httpbin.org/get") +// fmt.Println("Error:", err) +// fmt.Println("Trace Info:", resp.Request.TraceInfo()) +// +// Also `Request.EnableTrace` available too to get trace info for single request. +// +// Since v2.0.0 +func (c *Client) EnableTrace() *Client { + c.trace = true + return c +} + +// DisableTrace method disables the Resty client trace. Refer to `Client.EnableTrace`. +// +// Since v2.0.0 +func (c *Client) DisableTrace() *Client { + c.trace = false + return c +} + +// IsProxySet method returns the true is proxy is set from resty client otherwise +// false. By default proxy is set from environment, refer to `http.ProxyFromEnvironment`. +func (c *Client) IsProxySet() bool { + return c.proxyURL != nil +} + +// GetClient method returns the current `http.Client` used by the resty client. +func (c *Client) GetClient() *http.Client { + return c.httpClient +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Client Unexported methods +//_______________________________________________________________________ + +// Executes method executes the given `Request` object and returns response +// error. +func (c *Client) execute(req *Request) (*Response, error) { + defer releaseBuffer(req.bodyBuf) + // Apply Request middleware + var err error + + // user defined on before request methods + // to modify the *resty.Request object + for _, f := range c.udBeforeRequest { + if err = f(c, req); err != nil { + return nil, wrapNoRetryErr(err) + } + } + + // resty middlewares + for _, f := range c.beforeRequest { + if err = f(c, req); err != nil { + return nil, wrapNoRetryErr(err) + } + } + + if hostHeader := req.Header.Get("Host"); hostHeader != "" { + req.RawRequest.Host = hostHeader + } + + // call pre-request if defined + if c.preReqHook != nil { + if err = c.preReqHook(c, req.RawRequest); err != nil { + return nil, wrapNoRetryErr(err) + } + } + + if err = requestLogger(c, req); err != nil { + return nil, wrapNoRetryErr(err) + } + + req.Time = time.Now() + resp, err := c.httpClient.Do(req.RawRequest) + + response := &Response{ + Request: req, + RawResponse: resp, + } + + if err != nil || req.notParseResponse || c.notParseResponse { + response.setReceivedAt() + return response, err + } + + if !req.isSaveResponse { + defer closeq(resp.Body) + body := resp.Body + + // GitHub #142 & #187 + if strings.EqualFold(resp.Header.Get(hdrContentEncodingKey), "gzip") && resp.ContentLength != 0 { + if _, ok := body.(*gzip.Reader); !ok { + body, err = gzip.NewReader(body) + if err != nil { + response.setReceivedAt() + return response, err + } + defer closeq(body) + } + } + + if response.body, err = ioutil.ReadAll(body); err != nil { + response.setReceivedAt() + return response, err + } + + response.setReceivedAt() // after we read the body + response.size = int64(len(response.body)) + } + + // Apply Response middleware + for _, f := range c.afterResponse { + if err = f(c, response); err != nil { + break + } + } + + return response, wrapNoRetryErr(err) +} + +// getting TLS client config if not exists then create one +func (c *Client) tlsConfig() (*tls.Config, error) { + transport, err := c.transport() + if err != nil { + return nil, err + } + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + return transport.TLSClientConfig, nil +} + +// Transport method returns `*http.Transport` currently in use or error +// in case currently used `transport` is not a `*http.Transport`. +func (c *Client) transport() (*http.Transport, error) { + if transport, ok := c.httpClient.Transport.(*http.Transport); ok { + return transport, nil + } + return nil, errors.New("current transport is not an *http.Transport instance") +} + +// just an internal helper method +func (c *Client) outputLogTo(w io.Writer) *Client { + c.log.(*logger).l.SetOutput(w) + return c +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// File struct and its methods +//_______________________________________________________________________ + +// File struct represent file information for multipart request +type File struct { + Name string + ParamName string + io.Reader +} + +// String returns string value of current file details +func (f *File) String() string { + return fmt.Sprintf("ParamName: %v; FileName: %v", f.ParamName, f.Name) +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// MultipartField struct +//_______________________________________________________________________ + +// MultipartField struct represent custom data part for multipart request +type MultipartField struct { + Param string + FileName string + ContentType string + io.Reader +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Unexported package methods +//_______________________________________________________________________ + +func createClient(hc *http.Client) *Client { + if hc.Transport == nil { + hc.Transport = createTransport(nil) + } + + c := &Client{ // not setting lang default values + QueryParam: url.Values{}, + FormData: url.Values{}, + Header: http.Header{}, + Cookies: make([]*http.Cookie, 0), + RetryWaitTime: defaultWaitTime, + RetryMaxWaitTime: defaultMaxWaitTime, + JSONMarshal: json.Marshal, + JSONUnmarshal: json.Unmarshal, + jsonEscapeHTML: true, + httpClient: hc, + debugBodySizeLimit: math.MaxInt32, + pathParams: make(map[string]string), + } + + // Logger + c.SetLogger(createLogger()) + + // default before request middlewares + c.beforeRequest = []RequestMiddleware{ + parseRequestURL, + parseRequestHeader, + parseRequestBody, + createHTTPRequest, + addCredentials, + } + + // user defined request middlewares + c.udBeforeRequest = []RequestMiddleware{} + + // default after response middlewares + c.afterResponse = []ResponseMiddleware{ + responseLogger, + parseResponseBody, + saveResponseIntoFile, + } + + return c +} diff --git a/vendor/github.com/go-resty/resty/v2/go.mod b/vendor/github.com/go-resty/resty/v2/go.mod new file mode 100644 index 000000000..f1faf5f1b --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/go.mod @@ -0,0 +1,5 @@ +module github.com/go-resty/resty/v2 + +require golang.org/x/net v0.0.0-20200513185701-a91f0712d120 + +go 1.11 diff --git a/vendor/github.com/go-resty/resty/v2/middleware.go b/vendor/github.com/go-resty/resty/v2/middleware.go new file mode 100644 index 000000000..703f07c60 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/middleware.go @@ -0,0 +1,526 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/url" + "os" + "path/filepath" + "reflect" + "strings" + "time" +) + +const debugRequestLogKey = "__restyDebugRequestLog" + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Request Middleware(s) +//_______________________________________________________________________ + +func parseRequestURL(c *Client, r *Request) error { + // GitHub #103 Path Params + if len(r.pathParams) > 0 { + for p, v := range r.pathParams { + r.URL = strings.Replace(r.URL, "{"+p+"}", url.PathEscape(v), -1) + } + } + if len(c.pathParams) > 0 { + for p, v := range c.pathParams { + r.URL = strings.Replace(r.URL, "{"+p+"}", url.PathEscape(v), -1) + } + } + + // Parsing request URL + reqURL, err := url.Parse(r.URL) + if err != nil { + return err + } + + // If Request.URL is relative path then added c.HostURL into + // the request URL otherwise Request.URL will be used as-is + if !reqURL.IsAbs() { + r.URL = reqURL.String() + if len(r.URL) > 0 && r.URL[0] != '/' { + r.URL = "/" + r.URL + } + + reqURL, err = url.Parse(c.HostURL + r.URL) + if err != nil { + return err + } + } + + // Adding Query Param + query := make(url.Values) + for k, v := range c.QueryParam { + for _, iv := range v { + query.Add(k, iv) + } + } + + for k, v := range r.QueryParam { + // remove query param from client level by key + // since overrides happens for that key in the request + query.Del(k) + + for _, iv := range v { + query.Add(k, iv) + } + } + + // GitHub #123 Preserve query string order partially. + // Since not feasible in `SetQuery*` resty methods, because + // standard package `url.Encode(...)` sorts the query params + // alphabetically + if len(query) > 0 { + if IsStringEmpty(reqURL.RawQuery) { + reqURL.RawQuery = query.Encode() + } else { + reqURL.RawQuery = reqURL.RawQuery + "&" + query.Encode() + } + } + + r.URL = reqURL.String() + + return nil +} + +func parseRequestHeader(c *Client, r *Request) error { + hdr := make(http.Header) + for k := range c.Header { + hdr[k] = append(hdr[k], c.Header[k]...) + } + + for k := range r.Header { + hdr.Del(k) + hdr[k] = append(hdr[k], r.Header[k]...) + } + + if IsStringEmpty(hdr.Get(hdrUserAgentKey)) { + hdr.Set(hdrUserAgentKey, hdrUserAgentValue) + } + + ct := hdr.Get(hdrContentTypeKey) + if IsStringEmpty(hdr.Get(hdrAcceptKey)) && !IsStringEmpty(ct) && + (IsJSONType(ct) || IsXMLType(ct)) { + hdr.Set(hdrAcceptKey, hdr.Get(hdrContentTypeKey)) + } + + r.Header = hdr + + return nil +} + +func parseRequestBody(c *Client, r *Request) (err error) { + if isPayloadSupported(r.Method, c.AllowGetMethodPayload) { + // Handling Multipart + if r.isMultiPart && !(r.Method == MethodPatch) { + if err = handleMultipart(c, r); err != nil { + return + } + + goto CL + } + + // Handling Form Data + if len(c.FormData) > 0 || len(r.FormData) > 0 { + handleFormData(c, r) + + goto CL + } + + // Handling Request body + if r.Body != nil { + handleContentType(c, r) + + if err = handleRequestBody(c, r); err != nil { + return + } + } + } + +CL: + // by default resty won't set content length, you can if you want to :) + if (c.setContentLength || r.setContentLength) && r.bodyBuf != nil { + r.Header.Set(hdrContentLengthKey, fmt.Sprintf("%d", r.bodyBuf.Len())) + } + + return +} + +func createHTTPRequest(c *Client, r *Request) (err error) { + if r.bodyBuf == nil { + if reader, ok := r.Body.(io.Reader); ok { + r.RawRequest, err = http.NewRequest(r.Method, r.URL, reader) + } else { + r.RawRequest, err = http.NewRequest(r.Method, r.URL, nil) + } + } else { + r.RawRequest, err = http.NewRequest(r.Method, r.URL, r.bodyBuf) + } + + if err != nil { + return + } + + // Assign close connection option + r.RawRequest.Close = c.closeConnection + + // Add headers into http request + r.RawRequest.Header = r.Header + + // Add cookies from client instance into http request + for _, cookie := range c.Cookies { + r.RawRequest.AddCookie(cookie) + } + + // Add cookies from request instance into http request + for _, cookie := range r.Cookies { + r.RawRequest.AddCookie(cookie) + } + + // it's for non-http scheme option + if r.RawRequest.URL != nil && r.RawRequest.URL.Scheme == "" { + r.RawRequest.URL.Scheme = c.scheme + r.RawRequest.URL.Host = r.URL + } + + // Enable trace + if c.trace || r.trace { + r.clientTrace = &clientTrace{} + r.ctx = r.clientTrace.createContext(r.Context()) + } + + // Use context if it was specified + if r.ctx != nil { + r.RawRequest = r.RawRequest.WithContext(r.ctx) + } + + // assign get body func for the underlying raw request instance + r.RawRequest.GetBody = func() (io.ReadCloser, error) { + // If r.bodyBuf present, return the copy + if r.bodyBuf != nil { + return ioutil.NopCloser(bytes.NewReader(r.bodyBuf.Bytes())), nil + } + + // Maybe body is `io.Reader`. + // Note: Resty user have to watchout for large body size of `io.Reader` + if r.RawRequest.Body != nil { + b, err := ioutil.ReadAll(r.RawRequest.Body) + if err != nil { + return nil, err + } + + // Restore the Body + closeq(r.RawRequest.Body) + r.RawRequest.Body = ioutil.NopCloser(bytes.NewBuffer(b)) + + // Return the Body bytes + return ioutil.NopCloser(bytes.NewBuffer(b)), nil + } + + return nil, nil + } + + return +} + +func addCredentials(c *Client, r *Request) error { + var isBasicAuth bool + // Basic Auth + if r.UserInfo != nil { // takes precedence + r.RawRequest.SetBasicAuth(r.UserInfo.Username, r.UserInfo.Password) + isBasicAuth = true + } else if c.UserInfo != nil { + r.RawRequest.SetBasicAuth(c.UserInfo.Username, c.UserInfo.Password) + isBasicAuth = true + } + + if !c.DisableWarn { + if isBasicAuth && !strings.HasPrefix(r.URL, "https") { + c.log.Warnf("Using Basic Auth in HTTP mode is not secure, use HTTPS") + } + } + + // Set the Authorization Header Scheme + var authScheme string + if !IsStringEmpty(r.AuthScheme) { + authScheme = r.AuthScheme + } else if !IsStringEmpty(c.AuthScheme) { + authScheme = c.AuthScheme + } else { + authScheme = "Bearer" + } + + // Build the Token Auth header + if !IsStringEmpty(r.Token) { // takes precedence + r.RawRequest.Header.Set(hdrAuthorizationKey, authScheme+" "+r.Token) + } else if !IsStringEmpty(c.Token) { + r.RawRequest.Header.Set(hdrAuthorizationKey, authScheme+" "+c.Token) + } + + return nil +} + +func requestLogger(c *Client, r *Request) error { + if c.Debug { + rr := r.RawRequest + rl := &RequestLog{Header: copyHeaders(rr.Header), Body: r.fmtBodyString(c.debugBodySizeLimit)} + if c.requestLog != nil { + if err := c.requestLog(rl); err != nil { + return err + } + } + // fmt.Sprintf("COOKIES:\n%s\n", composeCookies(c.GetClient().Jar, *rr.URL)) + + + reqLog := "\n==============================================================================\n" + + "~~~ REQUEST ~~~\n" + + fmt.Sprintf("%s %s %s\n", r.Method, rr.URL.RequestURI(), rr.Proto) + + fmt.Sprintf("HOST : %s\n", rr.URL.Host) + + fmt.Sprintf("HEADERS:\n%s\n", composeHeaders(c, r, rl.Header)) + + fmt.Sprintf("BODY :\n%v\n", rl.Body) + + "------------------------------------------------------------------------------\n" + + r.initValuesMap() + r.values[debugRequestLogKey] = reqLog + } + + return nil +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Response Middleware(s) +//_______________________________________________________________________ + +func responseLogger(c *Client, res *Response) error { + if c.Debug { + rl := &ResponseLog{Header: copyHeaders(res.Header()), Body: res.fmtBodyString(c.debugBodySizeLimit)} + if c.responseLog != nil { + if err := c.responseLog(rl); err != nil { + return err + } + } + + debugLog := res.Request.values[debugRequestLogKey].(string) + debugLog += "~~~ RESPONSE ~~~\n" + + fmt.Sprintf("STATUS : %s\n", res.Status()) + + fmt.Sprintf("PROTO : %s\n", res.RawResponse.Proto) + + fmt.Sprintf("RECEIVED AT : %v\n", res.ReceivedAt().Format(time.RFC3339Nano)) + + fmt.Sprintf("TIME DURATION: %v\n", res.Time()) + + "HEADERS :\n" + + composeHeaders(c, res.Request, rl.Header) + "\n" + if res.Request.isSaveResponse { + debugLog += fmt.Sprintf("BODY :\n***** RESPONSE WRITTEN INTO FILE *****\n") + } else { + debugLog += fmt.Sprintf("BODY :\n%v\n", rl.Body) + } + debugLog += "==============================================================================\n" + + c.log.Debugf("%s", debugLog) + } + + return nil +} + +func parseResponseBody(c *Client, res *Response) (err error) { + if res.StatusCode() == http.StatusNoContent { + return + } + // Handles only JSON or XML content type + ct := firstNonEmpty(res.Request.forceContentType, res.Header().Get(hdrContentTypeKey), res.Request.fallbackContentType) + if IsJSONType(ct) || IsXMLType(ct) { + // HTTP status code > 199 and < 300, considered as Result + if res.IsSuccess() { + res.Request.Error = nil + if res.Request.Result != nil { + err = Unmarshalc(c, ct, res.body, res.Request.Result) + return + } + } + + // HTTP status code > 399, considered as Error + if res.IsError() { + // global error interface + if res.Request.Error == nil && c.Error != nil { + res.Request.Error = reflect.New(c.Error).Interface() + } + + if res.Request.Error != nil { + err = Unmarshalc(c, ct, res.body, res.Request.Error) + } + } + } + + return +} + +func handleMultipart(c *Client, r *Request) (err error) { + r.bodyBuf = acquireBuffer() + w := multipart.NewWriter(r.bodyBuf) + + for k, v := range c.FormData { + for _, iv := range v { + if err = w.WriteField(k, iv); err != nil { + return err + } + } + } + + for k, v := range r.FormData { + for _, iv := range v { + if strings.HasPrefix(k, "@") { // file + err = addFile(w, k[1:], iv) + if err != nil { + return + } + } else { // form value + if err = w.WriteField(k, iv); err != nil { + return err + } + } + } + } + + // #21 - adding io.Reader support + if len(r.multipartFiles) > 0 { + for _, f := range r.multipartFiles { + err = addFileReader(w, f) + if err != nil { + return + } + } + } + + // GitHub #130 adding multipart field support with content type + if len(r.multipartFields) > 0 { + for _, mf := range r.multipartFields { + if err = addMultipartFormField(w, mf); err != nil { + return + } + } + } + + r.Header.Set(hdrContentTypeKey, w.FormDataContentType()) + err = w.Close() + + return +} + +func handleFormData(c *Client, r *Request) { + formData := url.Values{} + + for k, v := range c.FormData { + for _, iv := range v { + formData.Add(k, iv) + } + } + + for k, v := range r.FormData { + // remove form data field from client level by key + // since overrides happens for that key in the request + formData.Del(k) + + for _, iv := range v { + formData.Add(k, iv) + } + } + + r.bodyBuf = bytes.NewBuffer([]byte(formData.Encode())) + r.Header.Set(hdrContentTypeKey, formContentType) + r.isFormData = true +} + +func handleContentType(c *Client, r *Request) { + contentType := r.Header.Get(hdrContentTypeKey) + if IsStringEmpty(contentType) { + contentType = DetectContentType(r.Body) + r.Header.Set(hdrContentTypeKey, contentType) + } +} + +func handleRequestBody(c *Client, r *Request) (err error) { + var bodyBytes []byte + contentType := r.Header.Get(hdrContentTypeKey) + kind := kindOf(r.Body) + r.bodyBuf = nil + + if reader, ok := r.Body.(io.Reader); ok { + if c.setContentLength || r.setContentLength { // keep backward compatibility + r.bodyBuf = acquireBuffer() + _, err = r.bodyBuf.ReadFrom(reader) + r.Body = nil + } else { + // Otherwise buffer less processing for `io.Reader`, sounds good. + return + } + } else if b, ok := r.Body.([]byte); ok { + bodyBytes = b + } else if s, ok := r.Body.(string); ok { + bodyBytes = []byte(s) + } else if IsJSONType(contentType) && + (kind == reflect.Struct || kind == reflect.Map || kind == reflect.Slice) { + bodyBytes, err = jsonMarshal(c, r, r.Body) + } else if IsXMLType(contentType) && (kind == reflect.Struct) { + bodyBytes, err = xml.Marshal(r.Body) + } + + if bodyBytes == nil && r.bodyBuf == nil { + err = errors.New("unsupported 'Body' type/value") + } + + // if any errors during body bytes handling, return it + if err != nil { + return + } + + // []byte into Buffer + if bodyBytes != nil && r.bodyBuf == nil { + r.bodyBuf = acquireBuffer() + _, _ = r.bodyBuf.Write(bodyBytes) + } + + return +} + +func saveResponseIntoFile(c *Client, res *Response) error { + if res.Request.isSaveResponse { + file := "" + + if len(c.outputDirectory) > 0 && !filepath.IsAbs(res.Request.outputFile) { + file += c.outputDirectory + string(filepath.Separator) + } + + file = filepath.Clean(file + res.Request.outputFile) + if err := createDirectory(filepath.Dir(file)); err != nil { + return err + } + + outFile, err := os.Create(file) + if err != nil { + return err + } + defer closeq(outFile) + + // io.Copy reads maximum 32kb size, it is perfect for large file download too + defer closeq(res.RawResponse.Body) + + written, err := io.Copy(outFile, res.RawResponse.Body) + if err != nil { + return err + } + + res.size = written + } + + return nil +} diff --git a/vendor/github.com/go-resty/resty/v2/redirect.go b/vendor/github.com/go-resty/resty/v2/redirect.go new file mode 100644 index 000000000..2976b7c5a --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/redirect.go @@ -0,0 +1,101 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "errors" + "fmt" + "net" + "net/http" + "strings" +) + +type ( + // RedirectPolicy to regulate the redirects in the resty client. + // Objects implementing the RedirectPolicy interface can be registered as + // + // Apply function should return nil to continue the redirect jounery, otherwise + // return error to stop the redirect. + RedirectPolicy interface { + Apply(req *http.Request, via []*http.Request) error + } + + // The RedirectPolicyFunc type is an adapter to allow the use of ordinary functions as RedirectPolicy. + // If f is a function with the appropriate signature, RedirectPolicyFunc(f) is a RedirectPolicy object that calls f. + RedirectPolicyFunc func(*http.Request, []*http.Request) error +) + +// Apply calls f(req, via). +func (f RedirectPolicyFunc) Apply(req *http.Request, via []*http.Request) error { + return f(req, via) +} + +// NoRedirectPolicy is used to disable redirects in the HTTP client +// resty.SetRedirectPolicy(NoRedirectPolicy()) +func NoRedirectPolicy() RedirectPolicy { + return RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { + return errors.New("auto redirect is disabled") + }) +} + +// FlexibleRedirectPolicy is convenient method to create No of redirect policy for HTTP client. +// resty.SetRedirectPolicy(FlexibleRedirectPolicy(20)) +func FlexibleRedirectPolicy(noOfRedirect int) RedirectPolicy { + return RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { + if len(via) >= noOfRedirect { + return fmt.Errorf("stopped after %d redirects", noOfRedirect) + } + checkHostAndAddHeaders(req, via[0]) + return nil + }) +} + +// DomainCheckRedirectPolicy is convenient method to define domain name redirect rule in resty client. +// Redirect is allowed for only mentioned host in the policy. +// resty.SetRedirectPolicy(DomainCheckRedirectPolicy("host1.com", "host2.org", "host3.net")) +func DomainCheckRedirectPolicy(hostnames ...string) RedirectPolicy { + hosts := make(map[string]bool) + for _, h := range hostnames { + hosts[strings.ToLower(h)] = true + } + + fn := RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { + if ok := hosts[getHostname(req.URL.Host)]; !ok { + return errors.New("redirect is not allowed as per DomainCheckRedirectPolicy") + } + + return nil + }) + + return fn +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Package Unexported methods +//_______________________________________________________________________ + +func getHostname(host string) (hostname string) { + if strings.Index(host, ":") > 0 { + host, _, _ = net.SplitHostPort(host) + } + hostname = strings.ToLower(host) + return +} + +// By default Golang will not redirect request headers +// after go throughing various discussion comments from thread +// https://github.com/golang/go/issues/4800 +// Resty will add all the headers during a redirect for the same host +func checkHostAndAddHeaders(cur *http.Request, pre *http.Request) { + curHostname := getHostname(cur.URL.Host) + preHostname := getHostname(pre.URL.Host) + if strings.EqualFold(curHostname, preHostname) { + for key, val := range pre.Header { + cur.Header[key] = val + } + } else { // only library User-Agent header is added + cur.Header.Set(hdrUserAgentKey, hdrUserAgentValue) + } +} diff --git a/vendor/github.com/go-resty/resty/v2/request.go b/vendor/github.com/go-resty/resty/v2/request.go new file mode 100644 index 000000000..3bf236fd9 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/request.go @@ -0,0 +1,809 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "net" + "net/http" + "net/url" + "reflect" + "strings" + "time" +) + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Request struct and methods +//_______________________________________________________________________ + +// Request struct is used to compose and fire individual request from +// resty client. Request provides an options to override client level +// settings and also an options for the request composition. +type Request struct { + URL string + Method string + Token string + AuthScheme string + QueryParam url.Values + FormData url.Values + Header http.Header + Time time.Time + Body interface{} + Result interface{} + Error interface{} + RawRequest *http.Request + SRV *SRVRecord + UserInfo *User + Cookies []*http.Cookie + + isMultiPart bool + isFormData bool + setContentLength bool + isSaveResponse bool + notParseResponse bool + jsonEscapeHTML bool + trace bool + outputFile string + fallbackContentType string + forceContentType string + ctx context.Context + pathParams map[string]string + values map[string]interface{} + client *Client + bodyBuf *bytes.Buffer + clientTrace *clientTrace + multipartFiles []*File + multipartFields []*MultipartField +} + +// Context method returns the Context if its already set in request +// otherwise it creates new one using `context.Background()`. +func (r *Request) Context() context.Context { + if r.ctx == nil { + return context.Background() + } + return r.ctx +} + +// SetContext method sets the context.Context for current Request. It allows +// to interrupt the request execution if ctx.Done() channel is closed. +// See https://blog.golang.org/context article and the "context" package +// documentation. +func (r *Request) SetContext(ctx context.Context) *Request { + r.ctx = ctx + return r +} + +// SetHeader method is to set a single header field and its value in the current request. +// +// For Example: To set `Content-Type` and `Accept` as `application/json`. +// client.R(). +// SetHeader("Content-Type", "application/json"). +// SetHeader("Accept", "application/json") +// +// Also you can override header value, which was set at client instance level. +func (r *Request) SetHeader(header, value string) *Request { + r.Header.Set(header, value) + return r +} + +// SetHeaders method sets multiple headers field and its values at one go in the current request. +// +// For Example: To set `Content-Type` and `Accept` as `application/json` +// +// client.R(). +// SetHeaders(map[string]string{ +// "Content-Type": "application/json", +// "Accept": "application/json", +// }) +// Also you can override header value, which was set at client instance level. +func (r *Request) SetHeaders(headers map[string]string) *Request { + for h, v := range headers { + r.SetHeader(h, v) + } + return r +} + +// SetQueryParam method sets single parameter and its value in the current request. +// It will be formed as query string for the request. +// +// For Example: `search=kitchen%20papers&size=large` in the URL after `?` mark. +// client.R(). +// SetQueryParam("search", "kitchen papers"). +// SetQueryParam("size", "large") +// Also you can override query params value, which was set at client instance level. +func (r *Request) SetQueryParam(param, value string) *Request { + r.QueryParam.Set(param, value) + return r +} + +// SetQueryParams method sets multiple parameters and its values at one go in the current request. +// It will be formed as query string for the request. +// +// For Example: `search=kitchen%20papers&size=large` in the URL after `?` mark. +// client.R(). +// SetQueryParams(map[string]string{ +// "search": "kitchen papers", +// "size": "large", +// }) +// Also you can override query params value, which was set at client instance level. +func (r *Request) SetQueryParams(params map[string]string) *Request { + for p, v := range params { + r.SetQueryParam(p, v) + } + return r +} + +// SetQueryParamsFromValues method appends multiple parameters with multi-value +// (`url.Values`) at one go in the current request. It will be formed as +// query string for the request. +// +// For Example: `status=pending&status=approved&status=open` in the URL after `?` mark. +// client.R(). +// SetQueryParamsFromValues(url.Values{ +// "status": []string{"pending", "approved", "open"}, +// }) +// Also you can override query params value, which was set at client instance level. +func (r *Request) SetQueryParamsFromValues(params url.Values) *Request { + for p, v := range params { + for _, pv := range v { + r.QueryParam.Add(p, pv) + } + } + return r +} + +// SetQueryString method provides ability to use string as an input to set URL query string for the request. +// +// Using String as an input +// client.R(). +// SetQueryString("productId=232&template=fresh-sample&cat=resty&source=google&kw=buy a lot more") +func (r *Request) SetQueryString(query string) *Request { + params, err := url.ParseQuery(strings.TrimSpace(query)) + if err == nil { + for p, v := range params { + for _, pv := range v { + r.QueryParam.Add(p, pv) + } + } + } else { + r.client.log.Errorf("%v", err) + } + return r +} + +// SetFormData method sets Form parameters and their values in the current request. +// It's applicable only HTTP method `POST` and `PUT` and requests content type would be set as +// `application/x-www-form-urlencoded`. +// client.R(). +// SetFormData(map[string]string{ +// "access_token": "BC594900-518B-4F7E-AC75-BD37F019E08F", +// "user_id": "3455454545", +// }) +// Also you can override form data value, which was set at client instance level. +func (r *Request) SetFormData(data map[string]string) *Request { + for k, v := range data { + r.FormData.Set(k, v) + } + return r +} + +// SetFormDataFromValues method appends multiple form parameters with multi-value +// (`url.Values`) at one go in the current request. +// client.R(). +// SetFormDataFromValues(url.Values{ +// "search_criteria": []string{"book", "glass", "pencil"}, +// }) +// Also you can override form data value, which was set at client instance level. +func (r *Request) SetFormDataFromValues(data url.Values) *Request { + for k, v := range data { + for _, kv := range v { + r.FormData.Add(k, kv) + } + } + return r +} + +// SetBody method sets the request body for the request. It supports various realtime needs as easy. +// We can say its quite handy or powerful. Supported request body data types is `string`, +// `[]byte`, `struct`, `map`, `slice` and `io.Reader`. Body value can be pointer or non-pointer. +// Automatic marshalling for JSON and XML content type, if it is `struct`, `map`, or `slice`. +// +// Note: `io.Reader` is processed as bufferless mode while sending request. +// +// For Example: Struct as a body input, based on content type, it will be marshalled. +// client.R(). +// SetBody(User{ +// Username: "jeeva@myjeeva.com", +// Password: "welcome2resty", +// }) +// +// Map as a body input, based on content type, it will be marshalled. +// client.R(). +// SetBody(map[string]interface{}{ +// "username": "jeeva@myjeeva.com", +// "password": "welcome2resty", +// "address": &Address{ +// Address1: "1111 This is my street", +// Address2: "Apt 201", +// City: "My City", +// State: "My State", +// ZipCode: 00000, +// }, +// }) +// +// String as a body input. Suitable for any need as a string input. +// client.R(). +// SetBody(`{ +// "username": "jeeva@getrightcare.com", +// "password": "admin" +// }`) +// +// []byte as a body input. Suitable for raw request such as file upload, serialize & deserialize, etc. +// client.R(). +// SetBody([]byte("This is my raw request, sent as-is")) +func (r *Request) SetBody(body interface{}) *Request { + r.Body = body + return r +} + +// SetResult method is to register the response `Result` object for automatic unmarshalling for the request, +// if response status code is between 200 and 299 and content type either JSON or XML. +// +// Note: Result object can be pointer or non-pointer. +// client.R().SetResult(&AuthToken{}) +// // OR +// client.R().SetResult(AuthToken{}) +// +// Accessing a result value from response instance. +// response.Result().(*AuthToken) +func (r *Request) SetResult(res interface{}) *Request { + r.Result = getPointer(res) + return r +} + +// SetError method is to register the request `Error` object for automatic unmarshalling for the request, +// if response status code is greater than 399 and content type either JSON or XML. +// +// Note: Error object can be pointer or non-pointer. +// client.R().SetError(&AuthError{}) +// // OR +// client.R().SetError(AuthError{}) +// +// Accessing a error value from response instance. +// response.Error().(*AuthError) +func (r *Request) SetError(err interface{}) *Request { + r.Error = getPointer(err) + return r +} + +// SetFile method is to set single file field name and its path for multipart upload. +// client.R(). +// SetFile("my_file", "/Users/jeeva/Gas Bill - Sep.pdf") +func (r *Request) SetFile(param, filePath string) *Request { + r.isMultiPart = true + r.FormData.Set("@"+param, filePath) + return r +} + +// SetFiles method is to set multiple file field name and its path for multipart upload. +// client.R(). +// SetFiles(map[string]string{ +// "my_file1": "/Users/jeeva/Gas Bill - Sep.pdf", +// "my_file2": "/Users/jeeva/Electricity Bill - Sep.pdf", +// "my_file3": "/Users/jeeva/Water Bill - Sep.pdf", +// }) +func (r *Request) SetFiles(files map[string]string) *Request { + r.isMultiPart = true + for f, fp := range files { + r.FormData.Set("@"+f, fp) + } + return r +} + +// SetFileReader method is to set single file using io.Reader for multipart upload. +// client.R(). +// SetFileReader("profile_img", "my-profile-img.png", bytes.NewReader(profileImgBytes)). +// SetFileReader("notes", "user-notes.txt", bytes.NewReader(notesBytes)) +func (r *Request) SetFileReader(param, fileName string, reader io.Reader) *Request { + r.isMultiPart = true + r.multipartFiles = append(r.multipartFiles, &File{ + Name: fileName, + ParamName: param, + Reader: reader, + }) + return r +} + +// SetMultipartFormData method allows simple form data to be attached to the request as `multipart:form-data` +func (r *Request) SetMultipartFormData(data map[string]string) *Request { + for k, v := range data { + r = r.SetMultipartField(k, "", "", strings.NewReader(v)) + } + + return r +} + +// SetMultipartField method is to set custom data using io.Reader for multipart upload. +func (r *Request) SetMultipartField(param, fileName, contentType string, reader io.Reader) *Request { + r.isMultiPart = true + r.multipartFields = append(r.multipartFields, &MultipartField{ + Param: param, + FileName: fileName, + ContentType: contentType, + Reader: reader, + }) + return r +} + +// SetMultipartFields method is to set multiple data fields using io.Reader for multipart upload. +// +// For Example: +// client.R().SetMultipartFields( +// &resty.MultipartField{ +// Param: "uploadManifest1", +// FileName: "upload-file-1.json", +// ContentType: "application/json", +// Reader: strings.NewReader(`{"input": {"name": "Uploaded document 1", "_filename" : ["file1.txt"]}}`), +// }, +// &resty.MultipartField{ +// Param: "uploadManifest2", +// FileName: "upload-file-2.json", +// ContentType: "application/json", +// Reader: strings.NewReader(`{"input": {"name": "Uploaded document 2", "_filename" : ["file2.txt"]}}`), +// }) +// +// If you have slice already, then simply call- +// client.R().SetMultipartFields(fields...) +func (r *Request) SetMultipartFields(fields ...*MultipartField) *Request { + r.isMultiPart = true + r.multipartFields = append(r.multipartFields, fields...) + return r +} + +// SetContentLength method sets the HTTP header `Content-Length` value for current request. +// By default Resty won't set `Content-Length`. Also you have an option to enable for every +// request. +// +// See `Client.SetContentLength` +// client.R().SetContentLength(true) +func (r *Request) SetContentLength(l bool) *Request { + r.setContentLength = true + return r +} + +// SetBasicAuth method sets the basic authentication header in the current HTTP request. +// +// For Example: +// Authorization: Basic +// +// To set the header for username "go-resty" and password "welcome" +// client.R().SetBasicAuth("go-resty", "welcome") +// +// This method overrides the credentials set by method `Client.SetBasicAuth`. +func (r *Request) SetBasicAuth(username, password string) *Request { + r.UserInfo = &User{Username: username, Password: password} + return r +} + +// SetAuthToken method sets the auth token header(Default Scheme: Bearer) in the current HTTP request. Header example: +// Authorization: Bearer +// +// For Example: To set auth token BC594900518B4F7EAC75BD37F019E08FBC594900518B4F7EAC75BD37F019E08F +// +// client.R().SetAuthToken("BC594900518B4F7EAC75BD37F019E08FBC594900518B4F7EAC75BD37F019E08F") +// +// This method overrides the Auth token set by method `Client.SetAuthToken`. +func (r *Request) SetAuthToken(token string) *Request { + r.Token = token + return r +} + +// SetAuthScheme method sets the auth token scheme type in the HTTP request. For Example: +// Authorization: +// +// For Example: To set the scheme to use OAuth +// +// client.R().SetAuthScheme("OAuth") +// +// This auth header scheme gets added to all the request rasied from this client instance. +// Also it can be overridden or set one at the request level is supported. +// +// Information about Auth schemes can be found in RFC7235 which is linked to below along with the page containing +// the currently defined official authentication schemes: +// https://tools.ietf.org/html/rfc7235 +// https://www.iana.org/assignments/http-authschemes/http-authschemes.xhtml#authschemes +// +// This method overrides the Authorization scheme set by method `Client.SetAuthScheme`. +func (r *Request) SetAuthScheme(scheme string) *Request { + r.AuthScheme = scheme + return r +} + +// SetOutput method sets the output file for current HTTP request. Current HTTP response will be +// saved into given file. It is similar to `curl -o` flag. Absolute path or relative path can be used. +// If is it relative path then output file goes under the output directory, as mentioned +// in the `Client.SetOutputDirectory`. +// client.R(). +// SetOutput("/Users/jeeva/Downloads/ReplyWithHeader-v5.1-beta.zip"). +// Get("http://bit.ly/1LouEKr") +// +// Note: In this scenario `Response.Body` might be nil. +func (r *Request) SetOutput(file string) *Request { + r.outputFile = file + r.isSaveResponse = true + return r +} + +// SetSRV method sets the details to query the service SRV record and execute the +// request. +// client.R(). +// SetSRV(SRVRecord{"web", "testservice.com"}). +// Get("/get") +func (r *Request) SetSRV(srv *SRVRecord) *Request { + r.SRV = srv + return r +} + +// SetDoNotParseResponse method instructs `Resty` not to parse the response body automatically. +// Resty exposes the raw response body as `io.ReadCloser`. Also do not forget to close the body, +// otherwise you might get into connection leaks, no connection reuse. +// +// Note: Response middlewares are not applicable, if you use this option. Basically you have +// taken over the control of response parsing from `Resty`. +func (r *Request) SetDoNotParseResponse(parse bool) *Request { + r.notParseResponse = parse + return r +} + +// SetPathParams method sets multiple URL path key-value pairs at one go in the +// Resty current request instance. +// client.R().SetPathParams(map[string]string{ +// "userId": "sample@sample.com", +// "subAccountId": "100002", +// }) +// +// Result: +// URL - /v1/users/{userId}/{subAccountId}/details +// Composed URL - /v1/users/sample@sample.com/100002/details +// It replace the value of the key while composing request URL. Also you can +// override Path Params value, which was set at client instance level. +func (r *Request) SetPathParams(params map[string]string) *Request { + for p, v := range params { + r.pathParams[p] = v + } + return r +} + +// ExpectContentType method allows to provide fallback `Content-Type` for automatic unmarshalling +// when `Content-Type` response header is unavailable. +func (r *Request) ExpectContentType(contentType string) *Request { + r.fallbackContentType = contentType + return r +} + +// ForceContentType method provides a strong sense of response `Content-Type` for automatic unmarshalling. +// Resty will respect it with higher priority; even response `Content-Type` response header value is available. +func (r *Request) ForceContentType(contentType string) *Request { + r.forceContentType = contentType + return r +} + +// SetJSONEscapeHTML method is to enable/disable the HTML escape on JSON marshal. +// +// Note: This option only applicable to standard JSON Marshaller. +func (r *Request) SetJSONEscapeHTML(b bool) *Request { + r.jsonEscapeHTML = b + return r +} + +// SetCookie method appends a single cookie in the current request instance. +// client.R().SetCookie(&http.Cookie{ +// Name:"go-resty", +// Value:"This is cookie value", +// }) +// +// Note: Method appends the Cookie value into existing Cookie if already existing. +// +// Since v2.1.0 +func (r *Request) SetCookie(hc *http.Cookie) *Request { + r.Cookies = append(r.Cookies, hc) + return r +} + +// SetCookies method sets an array of cookies in the current request instance. +// cookies := []*http.Cookie{ +// &http.Cookie{ +// Name:"go-resty-1", +// Value:"This is cookie 1 value", +// }, +// &http.Cookie{ +// Name:"go-resty-2", +// Value:"This is cookie 2 value", +// }, +// } +// +// // Setting a cookies into resty's current request +// client.R().SetCookies(cookies) +// +// Note: Method appends the Cookie value into existing Cookie if already existing. +// +// Since v2.1.0 +func (r *Request) SetCookies(rs []*http.Cookie) *Request { + r.Cookies = append(r.Cookies, rs...) + return r +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// HTTP request tracing +//_______________________________________________________________________ + +// EnableTrace method enables trace for the current request +// using `httptrace.ClientTrace` and provides insights. +// +// client := resty.New() +// +// resp, err := client.R().EnableTrace().Get("https://httpbin.org/get") +// fmt.Println("Error:", err) +// fmt.Println("Trace Info:", resp.Request.TraceInfo()) +// +// See `Client.EnableTrace` available too to get trace info for all requests. +// +// Since v2.0.0 +func (r *Request) EnableTrace() *Request { + r.trace = true + return r +} + +// TraceInfo method returns the trace info for the request. +// If either the Client or Request EnableTrace function has not been called +// prior to the request being made, an empty TraceInfo object will be returned. +// +// Since v2.0.0 +func (r *Request) TraceInfo() TraceInfo { + ct := r.clientTrace + + if ct == nil { + return TraceInfo{} + } + + ti := TraceInfo{ + DNSLookup: ct.dnsDone.Sub(ct.dnsStart), + TLSHandshake: ct.tlsHandshakeDone.Sub(ct.tlsHandshakeStart), + ServerTime: ct.gotFirstResponseByte.Sub(ct.gotConn), + TotalTime: ct.endTime.Sub(ct.dnsStart), + IsConnReused: ct.gotConnInfo.Reused, + IsConnWasIdle: ct.gotConnInfo.WasIdle, + ConnIdleTime: ct.gotConnInfo.IdleTime, + } + + // Only calcuate on successful connections + if !ct.connectDone.IsZero() { + ti.TCPConnTime = ct.connectDone.Sub(ct.dnsDone) + } + + // Only calcuate on successful connections + if !ct.gotConn.IsZero() { + ti.ConnTime = ct.gotConn.Sub(ct.getConn) + } + + // Only calcuate on successful connections + if !ct.gotFirstResponseByte.IsZero() { + ti.ResponseTime = ct.endTime.Sub(ct.gotFirstResponseByte) + } + + return ti +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// HTTP verb method starts here +//_______________________________________________________________________ + +// Get method does GET HTTP request. It's defined in section 4.3.1 of RFC7231. +func (r *Request) Get(url string) (*Response, error) { + return r.Execute(MethodGet, url) +} + +// Head method does HEAD HTTP request. It's defined in section 4.3.2 of RFC7231. +func (r *Request) Head(url string) (*Response, error) { + return r.Execute(MethodHead, url) +} + +// Post method does POST HTTP request. It's defined in section 4.3.3 of RFC7231. +func (r *Request) Post(url string) (*Response, error) { + return r.Execute(MethodPost, url) +} + +// Put method does PUT HTTP request. It's defined in section 4.3.4 of RFC7231. +func (r *Request) Put(url string) (*Response, error) { + return r.Execute(MethodPut, url) +} + +// Delete method does DELETE HTTP request. It's defined in section 4.3.5 of RFC7231. +func (r *Request) Delete(url string) (*Response, error) { + return r.Execute(MethodDelete, url) +} + +// Options method does OPTIONS HTTP request. It's defined in section 4.3.7 of RFC7231. +func (r *Request) Options(url string) (*Response, error) { + return r.Execute(MethodOptions, url) +} + +// Patch method does PATCH HTTP request. It's defined in section 2 of RFC5789. +func (r *Request) Patch(url string) (*Response, error) { + return r.Execute(MethodPatch, url) +} + +// Send method performs the HTTP request using the method and URL already defined +// for current `Request`. +// req := client.R() +// req.Method = resty.GET +// req.URL = "http://httpbin.org/get" +// resp, err := client.R().Send() +func (r *Request) Send() (*Response, error) { + return r.Execute(r.Method, r.URL) +} + +// Execute method performs the HTTP request with given HTTP method and URL +// for current `Request`. +// resp, err := client.R().Execute(resty.GET, "http://httpbin.org/get") +func (r *Request) Execute(method, url string) (*Response, error) { + var addrs []*net.SRV + var resp *Response + var err error + + if r.isMultiPart && !(method == MethodPost || method == MethodPut || method == MethodPatch) { + return nil, fmt.Errorf("multipart content is not allowed in HTTP verb [%v]", method) + } + + if r.SRV != nil { + _, addrs, err = net.LookupSRV(r.SRV.Service, "tcp", r.SRV.Domain) + if err != nil { + return nil, err + } + } + + r.Method = method + r.URL = r.selectAddr(addrs, url, 0) + + if r.client.RetryCount == 0 { + resp, err = r.client.execute(r) + return resp, unwrapNoRetryErr(err) + } + + attempt := 0 + err = Backoff( + func() (*Response, error) { + attempt++ + + r.URL = r.selectAddr(addrs, url, attempt) + + resp, err = r.client.execute(r) + if err != nil { + r.client.log.Errorf("%v, Attempt %v", err, attempt) + } + + return resp, err + }, + Retries(r.client.RetryCount), + WaitTime(r.client.RetryWaitTime), + MaxWaitTime(r.client.RetryMaxWaitTime), + RetryConditions(r.client.RetryConditions), + ) + + return resp, unwrapNoRetryErr(err) +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// SRVRecord struct +//_______________________________________________________________________ + +// SRVRecord struct holds the data to query the SRV record for the +// following service. +type SRVRecord struct { + Service string + Domain string +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Request Unexported methods +//_______________________________________________________________________ + +func (r *Request) fmtBodyString(sl int64) (body string) { + body = "***** NO CONTENT *****" + if !isPayloadSupported(r.Method, r.client.AllowGetMethodPayload) { + return + } + + if _, ok := r.Body.(io.Reader); ok { + body = "***** BODY IS io.Reader *****" + return + } + + // multipart or form-data + if r.isMultiPart || r.isFormData { + bodySize := int64(r.bodyBuf.Len()) + if bodySize > sl { + body = fmt.Sprintf("***** REQUEST TOO LARGE (size - %d) *****", bodySize) + return + } + body = r.bodyBuf.String() + return + } + + // request body data + if r.Body == nil { + return + } + var prtBodyBytes []byte + var err error + + contentType := r.Header.Get(hdrContentTypeKey) + kind := kindOf(r.Body) + if canJSONMarshal(contentType, kind) { + prtBodyBytes, err = json.MarshalIndent(&r.Body, "", " ") + } else if IsXMLType(contentType) && (kind == reflect.Struct) { + prtBodyBytes, err = xml.MarshalIndent(&r.Body, "", " ") + } else if b, ok := r.Body.(string); ok { + if IsJSONType(contentType) { + bodyBytes := []byte(b) + out := acquireBuffer() + defer releaseBuffer(out) + if err = json.Indent(out, bodyBytes, "", " "); err == nil { + prtBodyBytes = out.Bytes() + } + } else { + body = b + } + } else if b, ok := r.Body.([]byte); ok { + body = fmt.Sprintf("***** BODY IS byte(s) (size - %d) *****", len(b)) + return + } + + if prtBodyBytes != nil && err == nil { + body = string(prtBodyBytes) + } + + if len(body) > 0 { + bodySize := int64(len([]byte(body))) + if bodySize > sl { + body = fmt.Sprintf("***** REQUEST TOO LARGE (size - %d) *****", bodySize) + } + } + + return +} + +func (r *Request) selectAddr(addrs []*net.SRV, path string, attempt int) string { + if addrs == nil { + return path + } + + idx := attempt % len(addrs) + domain := strings.TrimRight(addrs[idx].Target, ".") + path = strings.TrimLeft(path, "/") + + return fmt.Sprintf("%s://%s:%d/%s", r.client.scheme, domain, addrs[idx].Port, path) +} + +func (r *Request) initValuesMap() { + if r.values == nil { + r.values = make(map[string]interface{}) + } +} + +var noescapeJSONMarshal = func(v interface{}) ([]byte, error) { + buf := acquireBuffer() + defer releaseBuffer(buf) + encoder := json.NewEncoder(buf) + encoder.SetEscapeHTML(false) + err := encoder.Encode(v) + return buf.Bytes(), err +} diff --git a/vendor/github.com/go-resty/resty/v2/response.go b/vendor/github.com/go-resty/resty/v2/response.go new file mode 100644 index 000000000..b82bce448 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/response.go @@ -0,0 +1,175 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Response struct and methods +//_______________________________________________________________________ + +// Response struct holds response values of executed request. +type Response struct { + Request *Request + RawResponse *http.Response + + body []byte + size int64 + receivedAt time.Time +} + +// Body method returns HTTP response as []byte array for the executed request. +// +// Note: `Response.Body` might be nil, if `Request.SetOutput` is used. +func (r *Response) Body() []byte { + if r.RawResponse == nil { + return []byte{} + } + return r.body +} + +// Status method returns the HTTP status string for the executed request. +// Example: 200 OK +func (r *Response) Status() string { + if r.RawResponse == nil { + return "" + } + return r.RawResponse.Status +} + +// StatusCode method returns the HTTP status code for the executed request. +// Example: 200 +func (r *Response) StatusCode() int { + if r.RawResponse == nil { + return 0 + } + return r.RawResponse.StatusCode +} + +// Proto method returns the HTTP response protocol used for the request. +func (r *Response) Proto() string { + if r.RawResponse == nil { + return "" + } + return r.RawResponse.Proto +} + +// Result method returns the response value as an object if it has one +func (r *Response) Result() interface{} { + return r.Request.Result +} + +// Error method returns the error object if it has one +func (r *Response) Error() interface{} { + return r.Request.Error +} + +// Header method returns the response headers +func (r *Response) Header() http.Header { + if r.RawResponse == nil { + return http.Header{} + } + return r.RawResponse.Header +} + +// Cookies method to access all the response cookies +func (r *Response) Cookies() []*http.Cookie { + if r.RawResponse == nil { + return make([]*http.Cookie, 0) + } + return r.RawResponse.Cookies() +} + +// String method returns the body of the server response as String. +func (r *Response) String() string { + if r.body == nil { + return "" + } + return strings.TrimSpace(string(r.body)) +} + +// Time method returns the time of HTTP response time that from request we sent and received a request. +// +// See `Response.ReceivedAt` to know when client recevied response and see `Response.Request.Time` to know +// when client sent a request. +func (r *Response) Time() time.Duration { + if r.Request.clientTrace != nil { + return r.Request.TraceInfo().TotalTime + } + return r.receivedAt.Sub(r.Request.Time) +} + +// ReceivedAt method returns when response got recevied from server for the request. +func (r *Response) ReceivedAt() time.Time { + return r.receivedAt +} + +// Size method returns the HTTP response size in bytes. Ya, you can relay on HTTP `Content-Length` header, +// however it won't be good for chucked transfer/compressed response. Since Resty calculates response size +// at the client end. You will get actual size of the http response. +func (r *Response) Size() int64 { + return r.size +} + +// RawBody method exposes the HTTP raw response body. Use this method in-conjunction with `SetDoNotParseResponse` +// option otherwise you get an error as `read err: http: read on closed response body`. +// +// Do not forget to close the body, otherwise you might get into connection leaks, no connection reuse. +// Basically you have taken over the control of response parsing from `Resty`. +func (r *Response) RawBody() io.ReadCloser { + if r.RawResponse == nil { + return nil + } + return r.RawResponse.Body +} + +// IsSuccess method returns true if HTTP status `code >= 200 and <= 299` otherwise false. +func (r *Response) IsSuccess() bool { + return r.StatusCode() > 199 && r.StatusCode() < 300 +} + +// IsError method returns true if HTTP status `code >= 400` otherwise false. +func (r *Response) IsError() bool { + return r.StatusCode() > 399 +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Response Unexported methods +//_______________________________________________________________________ + +func (r *Response) setReceivedAt() { + r.receivedAt = time.Now() + if r.Request.clientTrace != nil { + r.Request.clientTrace.endTime = r.receivedAt + } +} + +func (r *Response) fmtBodyString(sl int64) string { + if r.body != nil { + if int64(len(r.body)) > sl { + return fmt.Sprintf("***** RESPONSE TOO LARGE (size - %d) *****", len(r.body)) + } + ct := r.Header().Get(hdrContentTypeKey) + if IsJSONType(ct) { + out := acquireBuffer() + defer releaseBuffer(out) + err := json.Indent(out, r.body, "", " ") + if err != nil { + return fmt.Sprintf("*** Error: Unable to format response body - \"%s\" ***\n\nLog Body as-is:\n%s", err, r.String()) + } + return out.String() + } + return r.String() + } + + return "***** NO CONTENT *****" +} diff --git a/vendor/github.com/go-resty/resty/v2/resty.go b/vendor/github.com/go-resty/resty/v2/resty.go new file mode 100644 index 000000000..4685594b8 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/resty.go @@ -0,0 +1,40 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +// Package resty provides Simple HTTP and REST client library for Go. +package resty + +import ( + "net" + "net/http" + "net/http/cookiejar" + + "golang.org/x/net/publicsuffix" +) + +// Version # of resty +const Version = "2.3.0" + +// New method creates a new Resty client. +func New() *Client { + cookieJar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + return createClient(&http.Client{ + Jar: cookieJar, + }) +} + +// NewWithClient method creates a new Resty client with given `http.Client`. +func NewWithClient(hc *http.Client) *Client { + return createClient(hc) +} + +// NewWithLocalAddr method creates a new Resty client with given Local Address +// to dial from. +func NewWithLocalAddr(localAddr net.Addr) *Client { + cookieJar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + return createClient(&http.Client{ + Jar: cookieJar, + Transport: createTransport(localAddr), + }) +} diff --git a/vendor/github.com/go-resty/resty/v2/retry.go b/vendor/github.com/go-resty/resty/v2/retry.go new file mode 100644 index 000000000..0b7c6ffe8 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/retry.go @@ -0,0 +1,181 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "context" + "math" + "math/rand" + "time" +) + +const ( + defaultMaxRetries = 3 + defaultWaitTime = time.Duration(100) * time.Millisecond + defaultMaxWaitTime = time.Duration(2000) * time.Millisecond +) + +type ( + // Option is to create convenient retry options like wait time, max retries, etc. + Option func(*Options) + + // RetryConditionFunc type is for retry condition function + // input: non-nil Response OR request execution error + RetryConditionFunc func(*Response, error) bool + + // RetryAfterFunc returns time to wait before retry + // For example, it can parse HTTP Retry-After header + // https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + // Non-nil error is returned if it is found that request is not retryable + // (0, nil) is a special result means 'use default algorithm' + RetryAfterFunc func(*Client, *Response) (time.Duration, error) + + // Options struct is used to hold retry settings. + Options struct { + maxRetries int + waitTime time.Duration + maxWaitTime time.Duration + retryConditions []RetryConditionFunc + } +) + +// Retries sets the max number of retries +func Retries(value int) Option { + return func(o *Options) { + o.maxRetries = value + } +} + +// WaitTime sets the default wait time to sleep between requests +func WaitTime(value time.Duration) Option { + return func(o *Options) { + o.waitTime = value + } +} + +// MaxWaitTime sets the max wait time to sleep between requests +func MaxWaitTime(value time.Duration) Option { + return func(o *Options) { + o.maxWaitTime = value + } +} + +// RetryConditions sets the conditions that will be checked for retry. +func RetryConditions(conditions []RetryConditionFunc) Option { + return func(o *Options) { + o.retryConditions = conditions + } +} + +// Backoff retries with increasing timeout duration up until X amount of retries +// (Default is 3 attempts, Override with option Retries(n)) +func Backoff(operation func() (*Response, error), options ...Option) error { + // Defaults + opts := Options{ + maxRetries: defaultMaxRetries, + waitTime: defaultWaitTime, + maxWaitTime: defaultMaxWaitTime, + retryConditions: []RetryConditionFunc{}, + } + + for _, o := range options { + o(&opts) + } + + var ( + resp *Response + err error + ) + + for attempt := 0; attempt <= opts.maxRetries; attempt++ { + resp, err = operation() + ctx := context.Background() + if resp != nil && resp.Request.ctx != nil { + ctx = resp.Request.ctx + } + if ctx.Err() != nil { + return err + } + + err1 := unwrapNoRetryErr(err) // raw error, it used for return users callback. + needsRetry := err != nil && err == err1 // retry on a few operation errors by default + + for _, condition := range opts.retryConditions { + needsRetry = condition(resp, err1) + if needsRetry { + break + } + } + + if !needsRetry { + return err + } + + waitTime, err2 := sleepDuration(resp, opts.waitTime, opts.maxWaitTime, attempt) + if err2 != nil { + if err == nil { + err = err2 + } + return err + } + + select { + case <-time.After(waitTime): + case <-ctx.Done(): + return ctx.Err() + } + } + + return err +} + +func sleepDuration(resp *Response, min, max time.Duration, attempt int) (time.Duration, error) { + const maxInt = 1<<31 - 1 // max int for arch 386 + + if max < 0 { + max = maxInt + } + + if resp == nil { + goto defaultCase + } + + // 1. Check for custom callback + if retryAfterFunc := resp.Request.client.RetryAfter; retryAfterFunc != nil { + result, err := retryAfterFunc(resp.Request.client, resp) + if err != nil { + return 0, err // i.e. 'API quota exceeded' + } + if result == 0 { + goto defaultCase + } + if result < 0 || max < result { + result = max + } + if result < min { + result = min + } + return result, nil + } + + // 2. Return capped exponential backoff with jitter + // http://www.awsarchitectureblog.com/2015/03/backoff.html +defaultCase: + base := float64(min) + capLevel := float64(max) + + temp := math.Min(capLevel, base*math.Exp2(float64(attempt))) + ri := int(temp / 2) + if ri <= 0 { + ri = maxInt // max int for arch 386 + } + result := time.Duration(math.Abs(float64(ri + rand.Intn(ri)))) + + if result < min { + result = min + } + + return result, nil +} diff --git a/vendor/github.com/go-resty/resty/v2/trace.go b/vendor/github.com/go-resty/resty/v2/trace.go new file mode 100644 index 000000000..025b7d9b5 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/trace.go @@ -0,0 +1,122 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "context" + "crypto/tls" + "net/http/httptrace" + "time" +) + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// TraceInfo struct +//_______________________________________________________________________ + +// TraceInfo struct is used provide request trace info such as DNS lookup +// duration, Connection obtain duration, Server processing duration, etc. +// +// Since v2.0.0 +type TraceInfo struct { + // DNSLookup is a duration that transport took to perform + // DNS lookup. + DNSLookup time.Duration + + // ConnTime is a duration that took to obtain a successful connection. + ConnTime time.Duration + + // TCPConnTime is a duration that took to obtain the TCP connection. + TCPConnTime time.Duration + + // TLSHandshake is a duration that TLS handshake took place. + TLSHandshake time.Duration + + // ServerTime is a duration that server took to respond first byte. + ServerTime time.Duration + + // ResponseTime is a duration since first response byte from server to + // request completion. + ResponseTime time.Duration + + // TotalTime is a duration that total request took end-to-end. + TotalTime time.Duration + + // IsConnReused is whether this connection has been previously + // used for another HTTP request. + IsConnReused bool + + // IsConnWasIdle is whether this connection was obtained from an + // idle pool. + IsConnWasIdle bool + + // ConnIdleTime is a duration how long the connection was previously + // idle, if IsConnWasIdle is true. + ConnIdleTime time.Duration +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// CientTrace struct and its methods +//_______________________________________________________________________ + +// tracer struct maps the `httptrace.ClientTrace` hooks into Fields +// with same naming for easy understanding. Plus additional insights +// Request. +type clientTrace struct { + getConn time.Time + dnsStart time.Time + dnsDone time.Time + connectDone time.Time + tlsHandshakeStart time.Time + tlsHandshakeDone time.Time + gotConn time.Time + gotFirstResponseByte time.Time + endTime time.Time + gotConnInfo httptrace.GotConnInfo +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Trace unexported methods +//_______________________________________________________________________ + +func (t *clientTrace) createContext(ctx context.Context) context.Context { + return httptrace.WithClientTrace( + ctx, + &httptrace.ClientTrace{ + DNSStart: func(_ httptrace.DNSStartInfo) { + t.dnsStart = time.Now() + }, + DNSDone: func(_ httptrace.DNSDoneInfo) { + t.dnsDone = time.Now() + }, + ConnectStart: func(_, _ string) { + if t.dnsDone.IsZero() { + t.dnsDone = time.Now() + } + if t.dnsStart.IsZero() { + t.dnsStart = t.dnsDone + } + }, + ConnectDone: func(net, addr string, err error) { + t.connectDone = time.Now() + }, + GetConn: func(_ string) { + t.getConn = time.Now() + }, + GotConn: func(ci httptrace.GotConnInfo) { + t.gotConn = time.Now() + t.gotConnInfo = ci + }, + GotFirstResponseByte: func() { + t.gotFirstResponseByte = time.Now() + }, + TLSHandshakeStart: func() { + t.tlsHandshakeStart = time.Now() + }, + TLSHandshakeDone: func(_ tls.ConnectionState, _ error) { + t.tlsHandshakeDone = time.Now() + }, + }, + ) +} diff --git a/vendor/github.com/go-resty/resty/v2/transport.go b/vendor/github.com/go-resty/resty/v2/transport.go new file mode 100644 index 000000000..6cde29e8e --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/transport.go @@ -0,0 +1,35 @@ +// +build go1.13 + +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "net" + "net/http" + "runtime" + "time" +) + +func createTransport(localAddr net.Addr) *http.Transport { + dialer := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + } + if localAddr != nil { + dialer.LocalAddr = localAddr + } + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: dialer.DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + } +} diff --git a/vendor/github.com/go-resty/resty/v2/transport112.go b/vendor/github.com/go-resty/resty/v2/transport112.go new file mode 100644 index 000000000..ff7c2770c --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/transport112.go @@ -0,0 +1,34 @@ +// +build !go1.13 + +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "net" + "net/http" + "runtime" + "time" +) + +func createTransport(localAddr net.Addr) *http.Transport { + dialer := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + } + if localAddr != nil { + dialer.LocalAddr = localAddr + } + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: dialer.DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + } +} diff --git a/vendor/github.com/go-resty/resty/v2/util.go b/vendor/github.com/go-resty/resty/v2/util.go new file mode 100644 index 000000000..aaa53c236 --- /dev/null +++ b/vendor/github.com/go-resty/resty/v2/util.go @@ -0,0 +1,357 @@ +// Copyright (c) 2015-2020 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// resty source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package resty + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "log" + "mime/multipart" + "net/http" + "net/textproto" + "os" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" +) + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Logger interface +//_______________________________________________________________________ + +// Logger interface is to abstract the logging from Resty. Gives control to +// the Resty users, choice of the logger. +type Logger interface { + Errorf(format string, v ...interface{}) + Warnf(format string, v ...interface{}) + Debugf(format string, v ...interface{}) +} + +func createLogger() *logger { + l := &logger{l: log.New(os.Stderr, "", log.Ldate|log.Lmicroseconds)} + return l +} + +var _ Logger = (*logger)(nil) + +type logger struct { + l *log.Logger +} + +func (l *logger) Errorf(format string, v ...interface{}) { + l.output("ERROR RESTY "+format, v...) +} + +func (l *logger) Warnf(format string, v ...interface{}) { + l.output("WARN RESTY "+format, v...) +} + +func (l *logger) Debugf(format string, v ...interface{}) { + l.output("DEBUG RESTY "+format, v...) +} + +func (l *logger) output(format string, v ...interface{}) { + if len(v) == 0 { + l.l.Print(format) + return + } + l.l.Printf(format, v...) +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Package Helper methods +//_______________________________________________________________________ + +// IsStringEmpty method tells whether given string is empty or not +func IsStringEmpty(str string) bool { + return len(strings.TrimSpace(str)) == 0 +} + +// DetectContentType method is used to figure out `Request.Body` content type for request header +func DetectContentType(body interface{}) string { + contentType := plainTextType + kind := kindOf(body) + switch kind { + case reflect.Struct, reflect.Map: + contentType = jsonContentType + case reflect.String: + contentType = plainTextType + default: + if b, ok := body.([]byte); ok { + contentType = http.DetectContentType(b) + } else if kind == reflect.Slice { + contentType = jsonContentType + } + } + + return contentType +} + +// IsJSONType method is to check JSON content type or not +func IsJSONType(ct string) bool { + return jsonCheck.MatchString(ct) +} + +// IsXMLType method is to check XML content type or not +func IsXMLType(ct string) bool { + return xmlCheck.MatchString(ct) +} + +// Unmarshalc content into object from JSON or XML +func Unmarshalc(c *Client, ct string, b []byte, d interface{}) (err error) { + if IsJSONType(ct) { + err = c.JSONUnmarshal(b, d) + } else if IsXMLType(ct) { + err = xml.Unmarshal(b, d) + } + + return +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// RequestLog and ResponseLog type +//_______________________________________________________________________ + +// RequestLog struct is used to collected information from resty request +// instance for debug logging. It sent to request log callback before resty +// actually logs the information. +type RequestLog struct { + Header http.Header + Body string +} + +// ResponseLog struct is used to collected information from resty response +// instance for debug logging. It sent to response log callback before resty +// actually logs the information. +type ResponseLog struct { + Header http.Header + Body string +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Package Unexported methods +//_______________________________________________________________________ + +// way to disable the HTML escape as opt-in +func jsonMarshal(c *Client, r *Request, d interface{}) ([]byte, error) { + if !r.jsonEscapeHTML { + return noescapeJSONMarshal(d) + } else if !c.jsonEscapeHTML { + return noescapeJSONMarshal(d) + } + return c.JSONMarshal(d) +} + +func firstNonEmpty(v ...string) string { + for _, s := range v { + if !IsStringEmpty(s) { + return s + } + } + return "" +} + +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + +func escapeQuotes(s string) string { + return quoteEscaper.Replace(s) +} + +func createMultipartHeader(param, fileName, contentType string) textproto.MIMEHeader { + hdr := make(textproto.MIMEHeader) + + var contentDispositionValue string + if IsStringEmpty(fileName) { + contentDispositionValue = fmt.Sprintf(`form-data; name="%s"`, param) + } else { + contentDispositionValue = fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + param, escapeQuotes(fileName)) + } + hdr.Set("Content-Disposition", contentDispositionValue) + + if !IsStringEmpty(contentType) { + hdr.Set(hdrContentTypeKey, contentType) + } + return hdr +} + +func addMultipartFormField(w *multipart.Writer, mf *MultipartField) error { + partWriter, err := w.CreatePart(createMultipartHeader(mf.Param, mf.FileName, mf.ContentType)) + if err != nil { + return err + } + + _, err = io.Copy(partWriter, mf.Reader) + return err +} + +func writeMultipartFormFile(w *multipart.Writer, fieldName, fileName string, r io.Reader) error { + // Auto detect actual multipart content type + cbuf := make([]byte, 512) + size, err := r.Read(cbuf) + if err != nil { + return err + } + + partWriter, err := w.CreatePart(createMultipartHeader(fieldName, fileName, http.DetectContentType(cbuf))) + if err != nil { + return err + } + + if _, err = partWriter.Write(cbuf[:size]); err != nil { + return err + } + + _, err = io.Copy(partWriter, r) + return err +} + +func addFile(w *multipart.Writer, fieldName, path string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer closeq(file) + return writeMultipartFormFile(w, fieldName, filepath.Base(path), file) +} + +func addFileReader(w *multipart.Writer, f *File) error { + return writeMultipartFormFile(w, f.ParamName, f.Name, f.Reader) +} + +func getPointer(v interface{}) interface{} { + vv := valueOf(v) + if vv.Kind() == reflect.Ptr { + return v + } + return reflect.New(vv.Type()).Interface() +} + +func isPayloadSupported(m string, allowMethodGet bool) bool { + return !(m == MethodHead || m == MethodOptions || (m == MethodGet && !allowMethodGet)) +} + +func typeOf(i interface{}) reflect.Type { + return indirect(valueOf(i)).Type() +} + +func valueOf(i interface{}) reflect.Value { + return reflect.ValueOf(i) +} + +func indirect(v reflect.Value) reflect.Value { + return reflect.Indirect(v) +} + +func kindOf(v interface{}) reflect.Kind { + return typeOf(v).Kind() +} + +func createDirectory(dir string) (err error) { + if _, err = os.Stat(dir); err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(dir, 0755); err != nil { + return + } + } + } + return +} + +func canJSONMarshal(contentType string, kind reflect.Kind) bool { + return IsJSONType(contentType) && (kind == reflect.Struct || kind == reflect.Map || kind == reflect.Slice) +} + +func functionName(i interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() +} + +func acquireBuffer() *bytes.Buffer { + return bufPool.Get().(*bytes.Buffer) +} + +func releaseBuffer(buf *bytes.Buffer) { + if buf != nil { + buf.Reset() + bufPool.Put(buf) + } +} + +func closeq(v interface{}) { + if c, ok := v.(io.Closer); ok { + silently(c.Close()) + } +} + +func silently(_ ...interface{}) {} + +func composeHeaders(c *Client, r *Request, hdrs http.Header) string { + str := make([]string, 0, len(hdrs)) + for _, k := range sortHeaderKeys(hdrs) { + var v string + if k == "Cookie" { + cv := strings.TrimSpace(strings.Join(hdrs[k], ", ")) + if c.GetClient().Jar != nil { + for _, c := range c.GetClient().Jar.Cookies(r.RawRequest.URL) { + if cv != "" { + cv = cv + "; " + c.String() + } else { + cv = c.String() + } + } + } + v = strings.TrimSpace(fmt.Sprintf("%25s: %s", k, cv)) + } else { + v = strings.TrimSpace(fmt.Sprintf("%25s: %s", k, strings.Join(hdrs[k], ", "))) + } + if v != "" { + str = append(str, "\t"+v) + } + } + return strings.Join(str, "\n") +} + +func sortHeaderKeys(hdrs http.Header) []string { + keys := make([]string, 0, len(hdrs)) + for key := range hdrs { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func copyHeaders(hdrs http.Header) http.Header { + nh := http.Header{} + for k, v := range hdrs { + nh[k] = v + } + return nh +} + +type noRetryErr struct { + err error +} + +func (e *noRetryErr) Error() string { + return e.err.Error() +} + +func wrapNoRetryErr(err error) error { + if err != nil { + err = &noRetryErr{err: err} + } + return err +} + +func unwrapNoRetryErr(err error) error { + if e, ok := err.(*noRetryErr); ok { + err = e.err + } + return err +} From c09f4ec3cec65e5747b15ff53dd4a2812d820e06 Mon Sep 17 00:00:00 2001 From: colorfulberry Date: Mon, 20 Jul 2020 09:51:45 +0800 Subject: [PATCH 08/17] feat: improve the home --- templates/home.tmpl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/templates/home.tmpl b/templates/home.tmpl index 234c4f538..b946d4a1f 100644 --- a/templates/home.tmpl +++ b/templates/home.tmpl @@ -50,7 +50,7 @@

- +
@@ -90,7 +90,7 @@
- +
@@ -118,7 +118,7 @@
- +
@@ -134,7 +134,7 @@
- +
@@ -150,7 +150,7 @@
- +
@@ -177,7 +177,7 @@
- +
@@ -215,7 +215,7 @@ - +
From 38a45d7a4c0910023a46c3a4522e06d9fc28f1e2 Mon Sep 17 00:00:00 2001 From: palytoxin Date: Mon, 20 Jul 2020 09:55:18 +0800 Subject: [PATCH 09/17] add cloud brain config sample --- custom/conf/app.ini.sample | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index c7f096e53..c9bbe73ec 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -734,7 +734,7 @@ ALLOWED_TYPES = image/jpeg|image/png|application/zip|application/gzip MAX_SIZE = 4 ; Max number of files per upload. Defaults to 5 MAX_FILES = 5 -; Storage type for attachments, `local` for local disk or `minio` for s3 compitable +; Storage type for attachments, `local` for local disk or `minio` for s3 compitable ; object storage service, default is `local`. STORE_TYPE = local ; Path for attachments. Defaults to `data/attachments` only available when STORE_TYPE is `local` @@ -742,9 +742,9 @@ PATH = data/attachments ; Minio endpoint to connect only available when STORE_TYPE is `minio` MINIO_ENDPOINT = localhost:9000 ; Minio accessKeyID to connect only available when STORE_TYPE is `minio` -MINIO_ACCESS_KEY_ID = +MINIO_ACCESS_KEY_ID = ; Minio secretAccessKey to connect only available when STORE_TYPE is `minio` -MINIO_SECRET_ACCESS_KEY = +MINIO_SECRET_ACCESS_KEY = ; Minio bucket to store the attachments only available when STORE_TYPE is `minio` MINIO_BUCKET = gitea ; Minio location to create bucket only available when STORE_TYPE is `minio` @@ -1036,3 +1036,8 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" MAX_ATTEMPTS = 3 ; Backoff time per http/https request retry (seconds) RETRY_BACKOFF = 3 + +[cloudbrain] +HOST = http://192.168.204.24 +USERNAME = +PASSWORD = \ No newline at end of file From 1da042f251e780a26b1d74895dcc12f363b2df30 Mon Sep 17 00:00:00 2001 From: palytoxin Date: Mon, 20 Jul 2020 09:59:20 +0800 Subject: [PATCH 10/17] add show page --- models/cloudbrain.go | 74 ++--------------------------- modules/cloudbrain/cloudbrain.go | 3 +- modules/cloudbrain/resty.go | 2 +- routers/repo/cloudbrain.go | 18 +++++++ routers/routes/routes.go | 1 + templates/repo/cloudbrain/show.tmpl | 18 +++++++ 6 files changed, 45 insertions(+), 71 deletions(-) create mode 100644 templates/repo/cloudbrain/show.tmpl diff --git a/models/cloudbrain.go b/models/cloudbrain.go index dc97062da..ec0ad503f 100644 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -34,12 +34,7 @@ type Cloudbrain struct { type CloudBrainLoginResult struct { Code string Msg string - Payload struct { - UserID string `json:"userId"` - RealName string `json:"realName"` - Token string `json:"token"` - Admin bool `json:"admin"` - } + Payload map[string]interface{} } type TaskRole struct { @@ -67,72 +62,13 @@ type CreateJobParams struct { type CreateJobResult struct { Code string Msg string - Payload struct { - JobID string `json:"jobId"` - } + Payload map[string]interface{} } type GetJobResult struct { - Code string - Msg string - Payload struct { - ID string `json:"Id"` - Name string - Platform string - JobStatus struct { - Username string - State string - SubState string `json:"subState"` - ExecutionType string `json:"executionType"` - Retries int8 `json:"retries"` - CreatedTime int64 `json:"createdTime"` - CompletedTime int64 `json:"completedTime"` - AppID string `json:"appId"` - AppProgress string `json:"appProgress"` - AppTrackingURL string `json:"appTrackingUrl"` - AppLaunchedTime int64 `json:"appLaunchedTime"` - AppCompletedTime int64 `json:"appCompletedTime"` - AppExitCode int8 `json:"appExitCode"` - AppExitDiagnostics string `json:"appExitDiagnostics"` - AppExitType string `json:"appExitType"` - VirtualCluster string `json:"virtualCluster"` - } `json:"jobStatus"` - - TaskRoles string `json:"taskRoles"` - - Resource struct { - CPU int8 `json:"cpu"` - Memory string - GPU string `json:"nvidia.com/gpu"` - } `json:"resource"` - - Config struct { - Image string - JobID string `json:"jobId"` - GpuType string `json:"gpuType"` - JobName string `json:"jobName"` - JobType string `json:"jobType"` - RetryCount int8 `json:"retryCount"` - TaskRoles []struct { - Name string `json:"name"` - ShmMB int32 `json:"shmMB"` - Command string `json:"command"` - MemoryMB int64 `json:"memoryMB"` - CPUNumber int8 `json:"cpuNumber"` - GPUNumber int8 `json:"gpuNumber"` - IsMainRole bool `json:"isMainRole"` - TaskNumber int32 `json:"taskNumber"` - NeedIBDevice bool `json:"needIBDevice"` - MinFailedTaskCount int8 `json:"minFailedTaskCount"` - MinSucceededTaskCount int8 `json:"minSucceededTaskCount"` - } `json:"taskRoles"` - } - - Userinfo struct { - User string - OrgID string `json:"org_id"` - } - } + Code string `json:"code"` + Msg string `json:"msg"` + Payload map[string]interface{} `json:"payload"` } type CloudbrainsOptions struct { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index a015fa53a..606a29a14 100644 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -43,11 +43,12 @@ func GenerateTask(ctx *context.Context, image, command string) error { return errors.New(jobResult.Msg) } + var jobID = jobResult.Payload["jobId"].(string) err = models.CreateCloudbrain(&models.Cloudbrain{ Status: int32(models.JobWaiting), UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, - JobID: jobResult.Payload.JobID, + JobID: jobID, }) if err != nil { diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index 5f4551a8b..88134bab7 100644 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -51,7 +51,7 @@ func loginCloudbrain() error { return fmt.Errorf("%s: %s", loginResult.Msg, res.String()) } - TOKEN = loginResult.Payload.Token + TOKEN = loginResult.Payload["token"].(string) return nil } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d0667e151..6f20bc093 100644 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -12,6 +12,7 @@ import ( const ( tplCloudBrainIndex base.TplName = "repo/cloudbrain/index" tplCloudBrainNew base.TplName = "repo/cloudbrain/new" + tplCloudBrainShow base.TplName = "repo/cloudbrain/show" ) // MustEnableDataset check if repository enable internal cb @@ -70,3 +71,20 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") } + +func CloudBrainShow(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + var jobID = ctx.Params(":jobid") + var result, err = cloudbrain.GetJob(jobID) + if err != nil { + ctx.Data["error"] = err.Error() + } + + if result != nil { + ctx.Data["result"] = result + } + + ctx.Data["jobID"] = jobID + ctx.HTML(200, tplCloudBrainShow) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 4f66593de..e2cab421c 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -873,6 +873,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/cloudbrain", func() { m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) + m.Get("/:jobid", reqRepoCloudBrainReader, repo.CloudBrainShow) m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }, context.RepoRef()) diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl new file mode 100644 index 000000000..771934005 --- /dev/null +++ b/templates/repo/cloudbrain/show.tmpl @@ -0,0 +1,18 @@ +{{template "base/head" .}} +
+{{template "repo/header" .}} +
+
+ {{template "base/alert" .}} +
{{.jobID}}
+
+

err

+
{{.error}}
+
+

result

+
{{.result}}
+
+
+
+
+{{template "base/footer" .}} From 08a99e159b19c80cc70300a548ba4c7dbfabff49 Mon Sep 17 00:00:00 2001 From: palytoxin Date: Mon, 20 Jul 2020 14:36:36 +0800 Subject: [PATCH 11/17] add jobname to save model --- models/cloudbrain.go | 6 +++--- models/migrations/v141.go | 6 +++--- modules/auth/cloudbrain.go | 1 + modules/cloudbrain/cloudbrain.go | 19 +++++++------------ routers/repo/cloudbrain.go | 16 +++++++++++++++- templates/repo/cloudbrain/index.tmpl | 2 +- templates/repo/cloudbrain/new.tmpl | 5 +++++ 7 files changed, 35 insertions(+), 20 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index ec0ad503f..2485ef5d3 100644 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -18,9 +18,9 @@ const ( ) type Cloudbrain struct { - ID int64 `xorm:"pk autoincr"` - JobID string - // Title string `xorm:"INDEX NOT NULL"` + ID int64 `xorm:"pk autoincr"` + JobID string `xorm:"INDEX NOT NULL"` + JobName string Status int32 `xorm:"INDEX"` UserID int64 `xorm:"INDEX"` RepoID int64 `xorm:"INDEX"` diff --git a/models/migrations/v141.go b/models/migrations/v141.go index 45e9dc2b1..11a8d143b 100644 --- a/models/migrations/v141.go +++ b/models/migrations/v141.go @@ -9,9 +9,9 @@ import ( func addCloudBrainTable(x *xorm.Engine) error { type Cloudbrain struct { - ID int64 `xorm:"pk autoincr"` - JobId string - // Title string `xorm:"INDEX NOT NULL"` + ID int64 `xorm:"pk autoincr"` + JobID string `xorm:"INDEX NOT NULL"` + JobName string Status int32 `xorm:"INDEX"` UserID int64 `xorm:"INDEX"` RepoID int64 `xorm:"INDEX"` diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go index c77204937..938bf087e 100644 --- a/modules/auth/cloudbrain.go +++ b/modules/auth/cloudbrain.go @@ -7,6 +7,7 @@ import ( // CreateDatasetForm form for dataset page type CreateCloudBrainForm struct { + JobName string `form:"job_name" binding:"Required"` Image string `binding:"Required"` Command string `binding:"Required"` } diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 606a29a14..9f700c6d5 100644 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -2,19 +2,13 @@ package cloudbrain import ( "errors" - "fmt" - "strconv" - "time" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/models" ) -func GenerateTask(ctx *context.Context, image, command string) error { - nowStr := strconv.FormatInt(time.Now().Unix(), 10) - - jobName := fmt.Sprintf("%s%s", ctx.User.Name, nowStr[len(nowStr)-5:]) +func GenerateTask(ctx *context.Context, jobName, image, command string) error { jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -22,7 +16,7 @@ func GenerateTask(ctx *context.Context, image, command string) error { Image: image, TaskRoles: []models.TaskRole{ { - Name: jobName, + Name: "task1", TaskNumber: 1, MinSucceededTaskCount: 1, MinFailedTaskCount: 1, @@ -45,10 +39,11 @@ func GenerateTask(ctx *context.Context, image, command string) error { var jobID = jobResult.Payload["jobId"].(string) err = models.CreateCloudbrain(&models.Cloudbrain{ - Status: int32(models.JobWaiting), - UserID: ctx.User.ID, - RepoID: ctx.Repo.Repository.ID, - JobID: jobID, + Status: int32(models.JobWaiting), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: jobID, + JobName: jobName, }) if err != nil { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 6f20bc093..aaf1ed6da 100644 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -1,6 +1,9 @@ package repo import ( + "strconv" + "time" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -52,9 +55,19 @@ func CloudBrainIndex(ctx *context.Context) { ctx.HTML(200, tplCloudBrainIndex) } +func cutString(str string, lens int) string { + if len(str) < lens { + return str + } + return str[:lens] +} + func CloudBrainNew(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true + t := time.Now() + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[:5] + ctx.Data["job_name"] = jobName ctx.Data["image"] = "192.168.202.74:5000/user-images/deepo:v2.0" ctx.Data["command"] = `pip3 install jupyterlab==1.1.4;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir=\"/userhome\" --port=80 --NotebookApp.token=\"\" --LabApp.allow_origin=\"self https://cloudbrain.pcl.ac.cn\"` ctx.HTML(200, tplCloudBrainNew) @@ -62,9 +75,10 @@ func CloudBrainNew(ctx *context.Context) { func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.Data["PageIsCloudBrain"] = true + jobName := form.JobName image := form.Image command := form.Command - err := cloudbrain.GenerateTask(ctx, image, command) + err := cloudbrain.GenerateTask(ctx, jobName, image, command) if err != nil { ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index 4243d79a9..ca2c982d6 100644 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -36,7 +36,7 @@
{{svg "octicon-tasklist" 16}} - {{.JobID}} + {{.JobName}}
diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index 42d12ecf7..8b0afe8a7 100644 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -10,6 +10,11 @@ New Cloudbrain task
+
+
+ + +

From 35df6dbf439eed5ef29883b1620a3e5426cbc5da Mon Sep 17 00:00:00 2001 From: palytoxin Date: Mon, 20 Jul 2020 14:58:47 +0800 Subject: [PATCH 12/17] improve --- models/cloudbrain.go | 19 +++++++++++++++++++ routers/repo/cloudbrain.go | 8 +++++++- templates/repo/cloudbrain/show.tmpl | 12 +++++++++--- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 2485ef5d3..acfb37e08 100644 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1,6 +1,7 @@ package models import ( + "errors" "fmt" "code.gitea.io/gitea/modules/setting" @@ -75,6 +76,7 @@ type CloudbrainsOptions struct { ListOptions RepoID int64 // include all repos if empty UserID int64 + JobID int64 JobStatus CloudbrainStatus SortType string CloudbrainIDs []int64 @@ -97,6 +99,12 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) { ) } + if (opts.JobID) > 0 { + cond.And( + builder.Eq{"cloudbrain.job_id": opts.JobID}, + ) + } + switch opts.JobStatus { case JobWaiting: cond.And(builder.Eq{"cloudbrain.status": int(JobWaiting)}) @@ -144,3 +152,14 @@ func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { return nil } + +func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) { + cb := &Cloudbrain{JobID: jobID} + has, err := x.Get(cb) + if err != nil { + return nil, err + } else if !has { + return nil, errors.New("cloudbrain task is not found") + } + return cb, nil +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index aaf1ed6da..01910d93d 100644 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -90,7 +90,13 @@ func CloudBrainShow(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true var jobID = ctx.Params(":jobid") - var result, err = cloudbrain.GetJob(jobID) + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.Data["error"] = err.Error() + } + ctx.Data["task"] = task + + result, err := cloudbrain.GetJob(jobID) if err != nil { ctx.Data["error"] = err.Error() } diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index 771934005..29c2f5096 100644 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -4,12 +4,18 @@
{{template "base/alert" .}} -
{{.jobID}}
+ {{with .task}} +

jobname:

+
{{.JobName}}
-

err

+

jobid:

+
{{.JobID}}
+
+ {{end}} +

err:

{{.error}}
-

result

+

result:

{{.result}}
From c83397ac5685a8d1d859c11840e96b7574361905 Mon Sep 17 00:00:00 2001 From: palytoxin Date: Mon, 20 Jul 2020 15:37:31 +0800 Subject: [PATCH 13/17] parse task result --- models/cloudbrain.go | 91 +++++++++++++++++++++++++++++ routers/repo/cloudbrain.go | 6 +- templates/repo/cloudbrain/show.tmpl | 2 + 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index acfb37e08..f76363518 100644 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1,8 +1,10 @@ package models import ( + "encoding/json" "errors" "fmt" + "time" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -81,6 +83,95 @@ type CloudbrainsOptions struct { SortType string CloudbrainIDs []int64 } +type TaskPod struct { + TaskRoleStatus struct { + Name string `json:"name"` + } `json:"taskRoleStatus"` + TaskStatuses []struct { + TaskIndex int `json:"taskIndex"` + PodUID string `json:"podUid"` + PodIP string `json:"podIp"` + PodName string `json:"podName"` + ContainerID string `json:"containerId"` + ContainerIP string `json:"containerIp"` + ContainerGpus string `json:"containerGpus"` + State string `json:"state"` + StartAt time.Time `json:"startAt"` + FinishedAt time.Time `json:"finishedAt"` + ExitCode int `json:"exitCode"` + ExitDiagnostics string `json:"exitDiagnostics"` + RetriedCount int `json:"retriedCount"` + } `json:"taskStatuses"` +} + +func ConvertToTaskPod(input map[string]interface{}) (TaskPod, error) { + data, _ := json.Marshal(input) + var taskPod TaskPod + err := json.Unmarshal(data, &taskPod) + return taskPod, err +} + +type JobResultPayload struct { + ID string `json:"id"` + Name string `json:"name"` + Platform string `json:"platform"` + JobStatus struct { + Username string `json:"username"` + State string `json:"state"` + SubState string `json:"subState"` + ExecutionType string `json:"executionType"` + Retries int `json:"retries"` + CreatedTime int64 `json:"createdTime"` + CompletedTime int64 `json:"completedTime"` + AppID string `json:"appId"` + AppProgress string `json:"appProgress"` + AppTrackingURL string `json:"appTrackingUrl"` + AppLaunchedTime int64 `json:"appLaunchedTime"` + AppCompletedTime int64 `json:"appCompletedTime"` + AppExitCode int `json:"appExitCode"` + AppExitDiagnostics string `json:"appExitDiagnostics"` + AppExitType interface{} `json:"appExitType"` + VirtualCluster string `json:"virtualCluster"` + } `json:"jobStatus"` + TaskRoles map[string]interface{} `json:"taskRoles"` + Resource struct { + CPU int `json:"cpu"` + Memory string `json:"memory"` + NvidiaComGpu int `json:"nvidia.com/gpu"` + } `json:"resource"` + Config struct { + Image string `json:"image"` + JobID string `json:"jobId"` + GpuType string `json:"gpuType"` + JobName string `json:"jobName"` + JobType string `json:"jobType"` + TaskRoles []struct { + Name string `json:"name"` + ShmMB int `json:"shmMB"` + Command string `json:"command"` + MemoryMB int `json:"memoryMB"` + CPUNumber int `json:"cpuNumber"` + GpuNumber int `json:"gpuNumber"` + IsMainRole bool `json:"isMainRole"` + TaskNumber int `json:"taskNumber"` + NeedIBDevice bool `json:"needIBDevice"` + MinFailedTaskCount int `json:"minFailedTaskCount"` + MinSucceededTaskCount int `json:"minSucceededTaskCount"` + } `json:"taskRoles"` + RetryCount int `json:"retryCount"` + } `json:"config"` + Userinfo struct { + User string `json:"user"` + OrgID string `json:"org_id"` + } `json:"userinfo"` +} + +func ConvertToJobResultPayload(input map[string]interface{}) (JobResultPayload, error) { + data, _ := json.Marshal(input) + var jobResultPayload JobResultPayload + err := json.Unmarshal(data, &jobResultPayload) + return jobResultPayload, err +} func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) { sess := x.NewSession() diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 01910d93d..e258eadd7 100644 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -102,7 +102,11 @@ func CloudBrainShow(ctx *context.Context) { } if result != nil { - ctx.Data["result"] = result + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + ctx.Data["result"] = jobRes + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles["task1"].(map[string]interface{})) + ctx.Data["taskRes"] = taskRes } ctx.Data["jobID"] = jobID diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index 29c2f5096..29de55309 100644 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -12,6 +12,8 @@
{{.JobID}}
{{end}} +

taskRes:

+
{{.taskRes}}

err:

{{.error}}
From 11038143b53b841623994087b71b00db0bc02dbf Mon Sep 17 00:00:00 2001 From: palytoxin Date: Mon, 20 Jul 2020 17:28:04 +0800 Subject: [PATCH 14/17] change default gpuType --- modules/cloudbrain/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 9f700c6d5..2f2400d4b 100644 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -12,7 +12,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command string) error { jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, - GpuType: "debug", + GpuType: "dgx", Image: image, TaskRoles: []models.TaskRole{ { From fdd84ca9a6836c52239dc123f53433473ac30473 Mon Sep 17 00:00:00 2001 From: palytoxin Date: Tue, 21 Jul 2020 11:57:02 +0800 Subject: [PATCH 15/17] improve cb show page --- modules/templates/helper.go | 3 + templates/repo/cloudbrain/show.tmpl | 110 ++++++++++++++++++++++++---- 2 files changed, 97 insertions(+), 16 deletions(-) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 8971a3faf..d5d7bcdab 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -134,6 +134,9 @@ func NewFuncMap() []template.FuncMap { "EscapePound": func(str string) string { return strings.NewReplacer("%", "%25", "#", "%23", " ", "%20", "?", "%3F").Replace(str) }, + "nl2br": func(text string) template.HTML { + return template.HTML(strings.Replace(template.HTMLEscapeString(text), "\n", "
", -1)) + }, "PathEscapeSegments": util.PathEscapeSegments, "URLJoin": util.URLJoin, "RenderCommitMessage": RenderCommitMessage, diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index 29de55309..5afc00886 100644 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -4,22 +4,100 @@
{{template "base/alert" .}} - {{with .task}} -

jobname:

-
{{.JobName}}
-
-

jobid:

-
{{.JobID}}
-
- {{end}} -

taskRes:

-
{{.taskRes}}
-

err:

-
{{.error}}
-
-

result:

-
{{.result}}
-
+ +

+ 返回 +

+
+
+ {{with .task}} +

任务名称: {{.JobName}}

+ {{end}} +
+
+

任务结果:

+ {{with .taskRes}} + {{range .TaskStatuses}} + + + + + + + + + + + + + + + + + + + + + + + +
状态 {{.State}}
开始时间 {{.StartAt}}
结束时间 {{.FinishedAt}}
ExitCode {{.ExitCode}}
退出信息 {{.ExitDiagnostics| nl2br}}
+ {{end}} + {{end}} +
+
+ {{with .result}} + + + + + + + + + + + + + + + + + + +
硬件信息
CPU {{.Resource.CPU}}
Memory {{.Resource.Memory}}
NvidiaComGpu {{.Resource.NvidiaComGpu}}
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
调试信息
状态 {{.Platform}}
开始时间 {{.JobStatus.CreatedTime}}
结束时间 {{.JobStatus.CompletedTime}}
ExitCode {{.JobStatus.AppExitCode}}
退出信息 {{.JobStatus.AppExitDiagnostics | nl2br}}
+ {{end}} +
+
+
From 2e729899ec02ebbee2c2a3cc9819db811bf2c166 Mon Sep 17 00:00:00 2001 From: palytoxin Date: Tue, 21 Jul 2020 17:18:49 +0800 Subject: [PATCH 16/17] js fetch index job status --- models/cloudbrain.go | 47 ++++++++++------- models/migrations/v141.go | 2 +- modules/cloudbrain/cloudbrain.go | 2 +- routers/api/v1/api.go | 3 ++ routers/api/v1/repo/cloudbrain.go | 75 ++++++++++++++++++++++++++++ templates/repo/cloudbrain/index.tmpl | 39 +++++++++++---- templates/swagger/v1_json.tmpl | 38 ++++++++++++++ 7 files changed, 175 insertions(+), 31 deletions(-) create mode 100644 routers/api/v1/repo/cloudbrain.go diff --git a/models/cloudbrain.go b/models/cloudbrain.go index f76363518..12d17f56c 100644 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -11,20 +11,20 @@ import ( "xorm.io/builder" ) -type CloudbrainStatus int8 +type CloudbrainStatus string const ( - JobWaiting CloudbrainStatus = iota - JobStopped - JobSucceeded - JobFailed + JobWaiting CloudbrainStatus = "WAITING" + JobStopped CloudbrainStatus = "STOPPED" + JobSucceeded CloudbrainStatus = "SUCCEEDED" + JobFailed CloudbrainStatus = "FAILED" ) type Cloudbrain struct { ID int64 `xorm:"pk autoincr"` JobID string `xorm:"INDEX NOT NULL"` JobName string - Status int32 `xorm:"INDEX"` + Status string `xorm:"INDEX"` UserID int64 `xorm:"INDEX"` RepoID int64 `xorm:"INDEX"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` @@ -79,9 +79,9 @@ type CloudbrainsOptions struct { RepoID int64 // include all repos if empty UserID int64 JobID int64 - JobStatus CloudbrainStatus SortType string CloudbrainIDs []int64 + // JobStatus CloudbrainStatus } type TaskPod struct { TaskRoleStatus struct { @@ -127,7 +127,7 @@ type JobResultPayload struct { AppProgress string `json:"appProgress"` AppTrackingURL string `json:"appTrackingUrl"` AppLaunchedTime int64 `json:"appLaunchedTime"` - AppCompletedTime int64 `json:"appCompletedTime"` + AppCompletedTime interface{} `json:"appCompletedTime"` AppExitCode int `json:"appExitCode"` AppExitDiagnostics string `json:"appExitDiagnostics"` AppExitType interface{} `json:"appExitType"` @@ -196,16 +196,16 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) { ) } - switch opts.JobStatus { - case JobWaiting: - cond.And(builder.Eq{"cloudbrain.status": int(JobWaiting)}) - case JobFailed: - cond.And(builder.Eq{"cloudbrain.status": int(JobFailed)}) - case JobStopped: - cond.And(builder.Eq{"cloudbrain.status": int(JobStopped)}) - case JobSucceeded: - cond.And(builder.Eq{"cloudbrain.status": int(JobSucceeded)}) - } + // switch opts.JobStatus { + // case JobWaiting: + // cond.And(builder.Eq{"cloudbrain.status": int(JobWaiting)}) + // case JobFailed: + // cond.And(builder.Eq{"cloudbrain.status": int(JobFailed)}) + // case JobStopped: + // cond.And(builder.Eq{"cloudbrain.status": int(JobStopped)}) + // case JobSucceeded: + // cond.And(builder.Eq{"cloudbrain.status": int(JobSucceeded)}) + // } if len(opts.CloudbrainIDs) > 0 { cond.And(builder.In("cloudbrain.id", opts.CloudbrainIDs)) @@ -244,6 +244,17 @@ func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { return nil } +func GetRepoCloudBrainByJobID(repoID int64, jobID string) (*Cloudbrain, error) { + cb := &Cloudbrain{JobID: jobID, RepoID: repoID} + has, err := x.Get(cb) + if err != nil { + return nil, err + } else if !has { + return nil, errors.New("cloudbrain task is not found") + } + return cb, nil +} + func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) { cb := &Cloudbrain{JobID: jobID} has, err := x.Get(cb) diff --git a/models/migrations/v141.go b/models/migrations/v141.go index 11a8d143b..f8653f735 100644 --- a/models/migrations/v141.go +++ b/models/migrations/v141.go @@ -12,7 +12,7 @@ func addCloudBrainTable(x *xorm.Engine) error { ID int64 `xorm:"pk autoincr"` JobID string `xorm:"INDEX NOT NULL"` JobName string - Status int32 `xorm:"INDEX"` + Status string `xorm:"INDEX"` UserID int64 `xorm:"INDEX"` RepoID int64 `xorm:"INDEX"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 2f2400d4b..96a6f2957 100644 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -39,7 +39,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command string) error { var jobID = jobResult.Payload["jobId"].(string) err = models.CreateCloudbrain(&models.Cloudbrain{ - Status: int32(models.JobWaiting), + Status: string(models.JobWaiting), UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, JobID: jobID, diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 0d62b751c..064540a8c 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -849,6 +849,9 @@ func RegisterRoutes(m *macaron.Macaron) { Delete(reqToken(), repo.DeleteTopic) }, reqAdmin()) }, reqAnyRepoReader()) + m.Group("/cloudbrain", func() { + m.Get("/:jobid", repo.GetCloudbrainTask) + }, reqRepoReader(models.UnitTypeCloudBrain)) }, repoAssignment()) }) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go new file mode 100644 index 000000000..8a3aa067f --- /dev/null +++ b/routers/api/v1/repo/cloudbrain.go @@ -0,0 +1,75 @@ +// Copyright 2016 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repo + +import ( + "net/http" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/context" +) + +// cloudbrain get job task by jobid +func GetCloudbrainTask(ctx *context.APIContext) { + // swagger:operation GET /repos/{owner}/{repo}/cloudbrain/{jobid} cloudbrain jobTask + // --- + // summary: Get a single task + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: jobid + // in: path + // description: id of cloudbrain jobid + // type: string + // required: true + // responses: + // "200": + + // "$ref": "#/responses/Label" + + var ( + err error + ) + + jobID := ctx.Params(":jobid") + repoID := ctx.Repo.Repository.ID + _, err = models.GetRepoCloudBrainByJobID(repoID, jobID) + if err != nil { + ctx.NotFound(err) + return + } + jobResult, err := cloudbrain.GetJob(jobID) + if err != nil { + ctx.NotFound(err) + return + } + result, err := models.ConvertToJobResultPayload(jobResult.Payload) + if err != nil { + ctx.NotFound(err) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": result.Config.JobID, + "JobStatus": result.JobStatus.State, + "SubState": result.JobStatus.SubState, + "CreatedTime": time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"), + "CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"), + }) + +} diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index ca2c982d6..175b6aea9 100644 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -33,27 +33,24 @@ {{range .Tasks}}
-
+
{{svg "octicon-tasklist" 16}} {{.JobName}}
-
- waiting +
+ {{.Status}}
{{svg "octicon-flame" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}}
- -
- 18h 0m 8s -
- {{.Status}} -
-
- 再次提交 + + + 查看 + +
@@ -66,3 +63,23 @@
{{template "base/footer" .}} + diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index a6b24ed97..5745b5121 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2324,6 +2324,44 @@ } } }, + "/repos/{owner}/{repo}/cloudbrain/{jobid}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "cloudbrain" + ], + "summary": "Get a single task", + "operationId": "jobTask", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "id of cloudbrain jobid", + "name": "jobid", + "in": "path", + "required": true + } + ], + "responses": { + "200": {} + } + } + }, "/repos/{owner}/{repo}/collaborators": { "get": { "produces": [ From 851444ea30dd97d030a92386bd3464ebe1f8aea0 Mon Sep 17 00:00:00 2001 From: palytoxin Date: Wed, 22 Jul 2020 11:23:06 +0800 Subject: [PATCH 17/17] update job status when fetch job info --- models/cloudbrain.go | 22 +++++++++++++--------- routers/api/v1/repo/cloudbrain.go | 4 ++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 12d17f56c..731ae4196 100644 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -244,8 +244,7 @@ func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { return nil } -func GetRepoCloudBrainByJobID(repoID int64, jobID string) (*Cloudbrain, error) { - cb := &Cloudbrain{JobID: jobID, RepoID: repoID} +func getRepoCloudBrain(cb *Cloudbrain) (*Cloudbrain, error) { has, err := x.Get(cb) if err != nil { return nil, err @@ -255,13 +254,18 @@ func GetRepoCloudBrainByJobID(repoID int64, jobID string) (*Cloudbrain, error) { return cb, nil } +func GetRepoCloudBrainByJobID(repoID int64, jobID string) (*Cloudbrain, error) { + cb := &Cloudbrain{JobID: jobID, RepoID: repoID} + return getRepoCloudBrain(cb) +} + func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) { cb := &Cloudbrain{JobID: jobID} - has, err := x.Get(cb) - if err != nil { - return nil, err - } else if !has { - return nil, errors.New("cloudbrain task is not found") - } - return cb, nil + return getRepoCloudBrain(cb) +} + +func SetCloudbrainStatusByJobID(jobID string, status CloudbrainStatus) (err error) { + cb := &Cloudbrain{JobID: jobID, Status: string(status)} + _, err = x.Cols("status").Where("cloudbrain.job_id=?", jobID).Update(cb) + return } diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 8a3aa067f..56d41830b 100644 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -64,6 +64,10 @@ func GetCloudbrainTask(ctx *context.APIContext) { return } + if result.JobStatus.State != string(models.JobWaiting) { + go models.SetCloudbrainStatusByJobID(result.Config.JobID, models.CloudbrainStatus(result.JobStatus.State)) + } + ctx.JSON(http.StatusOK, map[string]interface{}{ "JobID": result.Config.JobID, "JobStatus": result.JobStatus.State,