#2282 fast detection of utf-8

gogs.go

@@ -17,7 +17,7 @@ import (
 	"github.com/gogits/gogs/modules/setting"
 )
 
-const APP_VER = "0.8.13.1225"
+const APP_VER = "0.8.13.1227"
 
 func init() {
 	runtime.GOMAXPROCS(runtime.NumCPU())

modules/base/tool.go

@@ -18,6 +18,7 @@ import (
 	"regexp"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/Unknwon/com"
 	"github.com/Unknwon/i18n"
@@ -53,6 +54,11 @@ func ShortSha(sha1 string) string {
 }
 
 func DetectEncoding(content []byte) string {
+	if utf8.Valid(content[:1024]) {
+		log.Debug("Detected encoding: utf-8 (fast)")
+		return "utf-8"
+	}
+
 	_, name, certain := charset.DetermineEncoding(content, "")
 	if name != "utf-8" && len(setting.Repository.AnsiCharset) > 0 {
 		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)

modules/template/template.go

@@ -12,7 +12,6 @@ import (
 	"runtime"
 	"strings"
 	"time"
-	"unicode/utf8"
 
 	"golang.org/x/net/html/charset"
 	"golang.org/x/text/transform"
@@ -131,11 +130,11 @@ func Sha1(str string) string {
 }
 
 func ToUtf8WithErr(content []byte) (error, string) {
-	if utf8.Valid(content[:1024]) {
+	charsetLabel := base.DetectEncoding(content)
+	if charsetLabel == "utf-8" {
 		return nil, string(content)
 	}
 
-	charsetLabel := base.DetectEncoding(content)
 	encoding, _ := charset.Lookup(charsetLabel)
 	if encoding == nil {
 		return fmt.Errorf("Unknown encoding: %s", charsetLabel), string(content)

templates/.VERSION

@@ -1 +1 @@
-0.8.13.1225
+0.8.13.1227