Implement "embedded" command to extract static resources (#9982)

* draft

* Implement extract command

* Fix nits and force args on extract

* Add !bindata stub, support Windows, fmt

* fix vendored flag

* Remove leading slash for matching

* Add docs

* Fix typos

* Add embedded view command

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

cmd/embedded.go

@@ -0,0 +1,332 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build bindata
+
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/options"
+	"code.gitea.io/gitea/modules/public"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/templates"
+
+	"github.com/gobwas/glob"
+	"github.com/urfave/cli"
+)
+
+// Cmdembedded represents the available extract sub-command.
+var (
+	Cmdembedded = cli.Command{
+		Name:        "embedded",
+		Usage:       "Extract embedded resources",
+		Description: "A command for extracting embedded resources, like templates and images",
+		Subcommands: []cli.Command{
+			subcmdList,
+			subcmdView,
+			subcmdExtract,
+		},
+	}
+
+	subcmdList = cli.Command{
+		Name:   "list",
+		Usage:  "List files matching the given pattern",
+		Action: runList,
+		Flags: []cli.Flag{
+			cli.BoolFlag{
+				Name:  "include-vendored,vendor",
+				Usage: "Include files under public/vendor as well",
+			},
+		},
+	}
+
+	subcmdView = cli.Command{
+		Name:   "view",
+		Usage:  "View a file matching the given pattern",
+		Action: runView,
+		Flags: []cli.Flag{
+			cli.BoolFlag{
+				Name:  "include-vendored,vendor",
+				Usage: "Include files under public/vendor as well",
+			},
+		},
+	}
+
+	subcmdExtract = cli.Command{
+		Name:   "extract",
+		Usage:  "Extract resources",
+		Action: runExtract,
+		Flags: []cli.Flag{
+			cli.BoolFlag{
+				Name:  "include-vendored,vendor",
+				Usage: "Include files under public/vendor as well",
+			},
+			cli.BoolFlag{
+				Name:  "overwrite",
+				Usage: "Overwrite files if they already exist",
+			},
+			cli.BoolFlag{
+				Name:  "rename",
+				Usage: "Rename files as {name}.bak if they already exist (overwrites previous .bak)",
+			},
+			cli.BoolFlag{
+				Name:  "custom",
+				Usage: "Extract to the 'custom' directory as per app.ini",
+			},
+			cli.StringFlag{
+				Name:  "destination,dest-dir",
+				Usage: "Extract to the specified directory",
+			},
+		},
+	}
+
+	sections map[string]*section
+	assets   []asset
+)
+
+type section struct {
+	Path  string
+	Names func() []string
+	IsDir func(string) (bool, error)
+	Asset func(string) ([]byte, error)
+}
+
+type asset struct {
+	Section *section
+	Name    string
+	Path    string
+}
+
+func initEmbeddedExtractor(c *cli.Context) error {
+
+	// Silence the console logger
+	log.DelNamedLogger("console")
+	log.DelNamedLogger(log.DEFAULT)
+
+	// Read configuration file
+	setting.NewContext()
+
+	pats, err := getPatterns(c.Args())
+	if err != nil {
+		return err
+	}
+	sections := make(map[string]*section, 3)
+
+	sections["public"] = &section{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset}
+	sections["options"] = &section{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset}
+	sections["templates"] = &section{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset}
+
+	for _, sec := range sections {
+		assets = append(assets, buildAssetList(sec, pats, c)...)
+	}
+
+	// Sort assets
+	sort.SliceStable(assets, func(i, j int) bool {
+		return assets[i].Path < assets[j].Path
+	})
+
+	return nil
+}
+
+func runList(c *cli.Context) error {
+	if err := runListDo(c); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		return err
+	}
+	return nil
+}
+
+func runView(c *cli.Context) error {
+	if err := runViewDo(c); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		return err
+	}
+	return nil
+}
+
+func runExtract(c *cli.Context) error {
+	if err := runExtractDo(c); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		return err
+	}
+	return nil
+}
+
+func runListDo(c *cli.Context) error {
+	if err := initEmbeddedExtractor(c); err != nil {
+		return err
+	}
+
+	for _, a := range assets {
+		fmt.Println(a.Path)
+	}
+
+	return nil
+}
+
+func runViewDo(c *cli.Context) error {
+	if err := initEmbeddedExtractor(c); err != nil {
+		return err
+	}
+
+	if len(assets) == 0 {
+		return fmt.Errorf("No files matched the given pattern")
+	} else if len(assets) > 1 {
+		return fmt.Errorf("Too many files matched the given pattern; try to be more specific")
+	}
+
+	data, err := assets[0].Section.Asset(assets[0].Name)
+	if err != nil {
+		return fmt.Errorf("%s: %v", assets[0].Path, err)
+	}
+
+	if _, err = os.Stdout.Write(data); err != nil {
+		return fmt.Errorf("%s: %v", assets[0].Path, err)
+	}
+
+	return nil
+}
+
+func runExtractDo(c *cli.Context) error {
+	if err := initEmbeddedExtractor(c); err != nil {
+		return err
+	}
+
+	if len(c.Args()) == 0 {
+		return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)")
+	}
+
+	destdir := "."
+
+	if c.IsSet("destination") {
+		destdir = c.String("destination")
+	} else if c.Bool("custom") {
+		destdir = setting.CustomPath
+		fmt.Println("Using app.ini at", setting.CustomConf)
+	}
+
+	fi, err := os.Stat(destdir)
+	if errors.Is(err, os.ErrNotExist) {
+		// In case Windows users attempt to provide a forward-slash path
+		wdestdir := filepath.FromSlash(destdir)
+		if wfi, werr := os.Stat(wdestdir); werr == nil {
+			destdir = wdestdir
+			fi = wfi
+			err = nil
+		}
+	}
+	if err != nil {
+		return fmt.Errorf("%s: %s", destdir, err)
+	} else if !fi.IsDir() {
+		return fmt.Errorf("%s is not a directory.", destdir)
+	}
+
+	fmt.Printf("Extracting to %s:\n", destdir)
+
+	overwrite := c.Bool("overwrite")
+	rename := c.Bool("rename")
+
+	for _, a := range assets {
+		if err := extractAsset(destdir, a, overwrite, rename); err != nil {
+			// Non-fatal error
+			fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err)
+		}
+	}
+
+	return nil
+}
+
+func extractAsset(d string, a asset, overwrite, rename bool) error {
+	dest := filepath.Join(d, filepath.FromSlash(a.Path))
+	dir := filepath.Dir(dest)
+
+	data, err := a.Section.Asset(a.Name)
+	if err != nil {
+		return fmt.Errorf("%s: %v", a.Path, err)
+	}
+
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		return fmt.Errorf("%s: %v", dir, err)
+	}
+
+	perms := os.ModePerm & 0666
+
+	fi, err := os.Lstat(dest)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("%s: %v", dest, err)
+		}
+	} else if !overwrite && !rename {
+		fmt.Printf("%s already exists; skipped.\n", dest)
+		return nil
+	} else if !fi.Mode().IsRegular() {
+		return fmt.Errorf("%s already exists, but it's not a regular file", dest)
+	} else if rename {
+		if err := os.Rename(dest, dest+".bak"); err != nil {
+			return fmt.Errorf("Error creating backup for %s: %v", dest, err)
+		}
+		// Attempt to respect file permissions mask (even if user:group will be set anew)
+		perms = fi.Mode()
+	}
+
+	file, err := os.OpenFile(dest, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, perms)
+	if err != nil {
+		return fmt.Errorf("%s: %v", dest, err)
+	}
+	defer file.Close()
+
+	if _, err = file.Write(data); err != nil {
+		return fmt.Errorf("%s: %v", dest, err)
+	}
+
+	fmt.Println(dest)
+
+	return nil
+}
+
+func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset {
+	var results = make([]asset, 0, 64)
+	for _, name := range sec.Names() {
+		if isdir, err := sec.IsDir(name); !isdir && err == nil {
+			if sec.Path == "public" &&
+				strings.HasPrefix(name, "vendor/") &&
+				!c.Bool("include-vendored") {
+				continue
+			}
+			matchName := sec.Path + "/" + name
+			for _, g := range globs {
+				if g.Match(matchName) {
+					results = append(results, asset{Section: sec,
+						Name: name,
+						Path: sec.Path + "/" + name})
+					break
+				}
+			}
+		}
+	}
+	return results
+}
+
+func getPatterns(args []string) ([]glob.Glob, error) {
+	if len(args) == 0 {
+		args = []string{"**"}
+	}
+	pat := make([]glob.Glob, len(args))
+	for i := range args {
+		if g, err := glob.Compile(args[i], '/'); err != nil {
+			return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err)
+		} else {
+			pat[i] = g
+		}
+	}
+	return pat, nil
+}

cmd/embedded_stub.go

@@ -0,0 +1,30 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !bindata
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli"
+)
+
+// Cmdembedded represents the available extract sub-command.
+var (
+	Cmdembedded = cli.Command{
+		Name:        "embedded",
+		Usage:       "Extract embedded resources",
+		Description: "A command for extracting embedded resources, like templates and images",
+		Action:      extractorNotImplemented,
+	}
+)
+
+func extractorNotImplemented(c *cli.Context) error {
+	err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata")
+	fmt.Fprintf(os.Stderr, "%s\n", err)
+	return err
+}

docs/content/doc/advanced/cmd-embedded.en-us.md

@@ -0,0 +1,115 @@
+---
+date: "2020-01-25T21:00:00-03:00"
+title: "Embedded data extraction tool"
+slug: "cmd-embedded"
+weight: 40
+toc: true
+draft: false
+menu:
+  sidebar:
+    parent: "advanced"
+    name: "Embedded data extraction tool"
+    weight: 40
+    identifier: "cmd-embedded"
+---
+
+# Embedded data extraction tool
+
+Gitea's executable contains all the resources required to run: templates, images, style-sheets
+and translations. Any of them can be overridden by placing a replacement in a matching path
+inside the `custom` directory (see [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}})).
+
+To obtain a copy of the embedded resources ready for editing, the `embedded` command from the CLI
+can be used from the OS shell interface.
+
+## Listing resources
+
+To list resources embedded in Gitea's executable, use the following syntax:
+
+```
+gitea embedded list [--include-vendored] [patterns...]
+```
+
+The `--include-vendored` flag makes the command include vendored files, which are
+normally excluded; that is, files from external libraries that are required for Gitea
+(e.g. [font-awesome](https://fontawesome.com/), [octicons](https://octicons.github.com/), etc).
+
+A list of file search patterns can be provided. Gitea uses [gobwas/glob](https://github.com/gobwas/glob)
+for its glob syntax. Here are some examples:
+
+- List all template files, in any virtual directory: `**.tmpl`
+- List all mail template files: `templates/mail/**.tmpl`
+- List all files inside `public/img`: `public/img/**`
+
+Don't forget to use quotes for the patterns, as spaces, `*` and other characters might have
+a special meaning for your command shell.
+
+If no pattern is provided, all files are listed.
+
+#### Example
+
+Listing all embedded files with `openid` in their path:
+
+```
+$ gitea embedded list '**openid**'
+public/img/auth/openid_connect.png
+public/img/openid-16x16.png
+templates/user/auth/finalize_openid.tmpl
+templates/user/auth/signin_openid.tmpl
+templates/user/auth/signup_openid_connect.tmpl
+templates/user/auth/signup_openid_navbar.tmpl
+templates/user/auth/signup_openid_register.tmpl
+templates/user/settings/security_openid.tmpl
+```
+
+## Extracting resources
+
+To extract resources embedded in Gitea's executable, use the following syntax:
+
+```
+gitea [--config {file}] embedded extract [--destination {dir}|--custom] [--overwrite|--rename] [--include-vendored] {patterns...}
+```
+
+The `--config` option tells gitea the location of the `app.ini` configuration file if
+it's not in its default location. This option is only used with the `--custom` flag.
+
+The `--destination` option tells gitea the directory where the files must be extracted to.
+The default is the current directory.
+
+The `--custom` flag tells gitea to extract the files directly into the `custom` directory.
+For this to work, the command needs to know the location of the `app.ini` configuration
+file (`--config`) and, depending of the configuration, be ran from the directory where
+gitea normally starts. See [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}}) for details.
+
+The `--overwrite` flag allows any existing files in the destination directory to be overwritten.
+
+The `--rename` flag tells gitea to rename any existing files in the destination directory
+as `filename.bak`. Previous `.bak` files are overwritten.
+
+At least one file search pattern must be provided; see `list` subcomand above for pattern
+syntax and examples.
+
+#### Important notice
+
+Make sure to **only extract those files that require customization**. Files that
+are present in the `custom` directory are not upgraded by Gitea's upgrade process.
+When Gitea is upgraded to a new version (by replacing the executable), many of the
+embedded files will suffer changes. Gitea will honor and use any files found
+in the `custom` directory, even if they are old and incompatible.
+
+#### Example
+
+Extracting mail templates to a temporary directory:
+
+```
+$ mkdir tempdir
+$ gitea embedded extract --destination tempdir 'templates/mail/**.tmpl'
+Extracting to tempdir:
+tempdir/templates/mail/auth/activate.tmpl
+tempdir/templates/mail/auth/activate_email.tmpl
+tempdir/templates/mail/auth/register_notify.tmpl
+tempdir/templates/mail/auth/reset_passwd.tmpl
+tempdir/templates/mail/issue/assigned.tmpl
+tempdir/templates/mail/issue/default.tmpl
+tempdir/templates/mail/notify/collaborator.tmpl
+```

docs/content/doc/advanced/customizing-gitea.en-us.md

@@ -57,14 +57,21 @@ the url `http://gitea.domain.tld/image.png`.
 
 Place the png image at the following path: `custom/public/img/avatar_default.png`
 
-## Customizing Gitea pages
+## Customizing Gitea pages and resources
 
-The `custom/templates` folder allows changing every single page of Gitea. Templates
-to override can be found in the [`templates`](https://github.com/go-gitea/gitea/tree/master/templates) directory of Gitea source (Note: the example link is from `master` branch. Make sure to copy templates from same release you are using). Override by
-making a copy of the file under `custom/templates` using a full path structure
-matching source.
+Gitea's executable contains all the resources required to run: templates, images, style-sheets
+and translations. Any of them can be overridden by placing a replacement in a matching path
+inside the `custom` directory. For example, to replace the default `.gitignore` provided
+for C++ repositories, we want to replace `options/gitignore/C++`. To do this, a replacement
+must be placed in `custom/options/gitignore/C++` (see about the location of the `custom`
+directory at the top of this document).
 
-Any statement contained inside `{{` and `}}` are Gitea's template syntax and
+Every single page of Gitea can be changed. Dynamic content is generated using [go templates](https://golang.org/pkg/html/template/),
+which can be modified by placing replacements below the `custom/templates` directory.
+
+To obtain any embedded file (including templates), the [`gitea embedded` tool]({{< relref "doc/advanced/cmd-embedded.en-us.md" >}}) can be used. Alternatively, they can be found in the [`templates`](https://github.com/go-gitea/gitea/tree/master/templates) directory of Gitea source (Note: the example link is from the `master` branch. Make sure to use templates compatible with the release you are using).
+
+Be aware that any statement contained inside `{{` and `}}` are Gitea's template syntax and
 shouldn't be touched without fully understanding these components.
 
 ### Customizing startpage / homepage

main.go

@@ -70,6 +70,7 @@ arguments - which can alternatively be run by running the subcommand web.`
 		cmd.CmdConvert,
 		cmd.CmdDoctor,
 		cmd.CmdManager,
+		cmd.Cmdembedded,
 	}
 	// Now adjust these commands to add our global configuration options
 

modules/options/static.go

@@ -113,6 +113,37 @@ func fileFromDir(name string) ([]byte, error) {
 	return ioutil.ReadAll(f)
 }
 
+func Asset(name string) ([]byte, error) {
+	f, err := Assets.Open("/" + name)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return ioutil.ReadAll(f)
+}
+
+func AssetNames() []string {
+	realFS := Assets.(vfsgen۰FS)
+	var results = make([]string, 0, len(realFS))
+	for k := range realFS {
+		results = append(results, k[1:])
+	}
+	return results
+}
+
+func AssetIsDir(name string) (bool, error) {
+	if f, err := Assets.Open("/" + name); err != nil {
+		return false, err
+	} else {
+		defer f.Close()
+		if fi, err := f.Stat(); err != nil {
+			return false, err
+		} else {
+			return fi.IsDir(), nil
+		}
+	}
+}
+
 // IsDynamic will return false when using embedded data (-tags bindata)
 func IsDynamic() bool {
 	return false

modules/public/static.go

@@ -7,6 +7,8 @@
 package public
 
 import (
+	"io/ioutil"
+
 	"gitea.com/macaron/macaron"
 )
 
@@ -17,3 +19,34 @@ func Static(opts *Options) macaron.Handler {
 	// used when in the options there is no FileSystem.
 	return opts.staticHandler("")
 }
+
+func Asset(name string) ([]byte, error) {
+	f, err := Assets.Open("/" + name)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return ioutil.ReadAll(f)
+}
+
+func AssetNames() []string {
+	realFS := Assets.(vfsgen۰FS)
+	var results = make([]string, 0, len(realFS))
+	for k := range realFS {
+		results = append(results, k[1:])
+	}
+	return results
+}
+
+func AssetIsDir(name string) (bool, error) {
+	if f, err := Assets.Open("/" + name); err != nil {
+		return false, err
+	} else {
+		defer f.Close()
+		if fi, err := f.Stat(); err != nil {
+			return false, err
+		} else {
+			return fi.IsDir(), nil
+		}
+	}
+}

modules/templates/static.go

@@ -229,3 +229,16 @@ func AssetNames() []string {
 	}
 	return results
 }
+
+func AssetIsDir(name string) (bool, error) {
+	if f, err := Assets.Open("/" + name); err != nil {
+		return false, err
+	} else {
+		defer f.Close()
+		if fi, err := f.Stat(); err != nil {
+			return false, err
+		} else {
+			return fi.IsDir(), nil
+		}
+	}
+}