Support custom robots.txt

cmd/web.go

@@ -364,6 +364,15 @@ func runWeb(*cli.Context) {
 		r.Any("/:reponame/*", ignSignInAndCsrf, repo.Http)
 	})
 
+	// robots.txt
+	m.Get("/robots.txt", func(ctx *middleware.Context) {
+		if setting.HasRobotsTxt {
+			ctx.ServeFile(path.Join(setting.CustomPath, "robots.txt"))
+		} else {
+			ctx.Error(404)
+		}
+	})
+
 	// Not found handler.
 	m.NotFound(routers.NotFound)
 

modules/setting/setting.go

@@ -108,6 +108,7 @@ var (
 	ProdMode     bool
 	RunUser      string
 	IsWindows    bool
+	HasRobotsTxt bool
 )
 
 func init() {
@@ -260,6 +261,8 @@ func NewConfigContext() {
 
 	Langs = Cfg.MustValueArray("i18n", "LANGS", ",")
 	Names = Cfg.MustValueArray("i18n", "NAMES", ",")
+
+	HasRobotsTxt = com.IsFile(path.Join(CustomPath, "robots.txt"))
 }
 
 var Service struct {