toys/adtools-light/adtools/adgit.py

#!/usr/bin/env python3
# -*- coding: utf-8; mode: python; tab-width: 4; indent-tabs-mode: nil -*-
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8
#
# THIS FILE IS PART OF adtools PROJECT
#
#           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
#                   Version 2, December 2004
#
# Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
#
# Everyone is permitted to copy and distribute verbatim or modified
# copies of this license document, and changing it is allowed as long
# as the name is changed.
#
#           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
#  TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
#
# 0. You just DO WHAT THE FUCK YOU WANT TO.
#
# Copyright (C) 2021- donkey <anjingyu_ws@foxmail.com>


import os
import sys
import json
import datetime
import argparse
import textwrap
import uuid
from urllib.parse import quote_plus, urlparse

import crayons

from .__init__ import __version__
from .adutil import AdUtil, AdGitHelper, E, W, D

__author__ = ['"donkey" <anjingyu_ws@foxmail.com>']


def __print_action(subcmd: str, name: str):
    print(crayons.green("=== do "), end="")
    print(crayons.yellow(subcmd), end="")
    print(crayons.green(" on repo "), end="")
    print(crayons.cyan(name), end="")
    print(crayons.green(" ===\n"))


def __do_action(git_helper, scmd: str, args: list):
    # Patch for push
    if scmd == "push":
        branch = git_helper.current_branch()
        if not args and not git_helper.has_remote_branch(branch):
            args.extend(["-u", "origin", branch])

    subcmd = "{} {}".format(scmd, " ".join(args)).strip()

    code, ret = git_helper.command(subcmd)
    if code != 0:
        W(f"Failed to run command <{subcmd}>\n{ret}")
    else:
        print(ret)


def __common_git_cmd(git_helper, args, extra_args):
    wd = os.path.abspath(".")
    print("{}: {}".format(crayons.yellow("Current workspace"), crayons.green(wd)))

    for dname in os.listdir(wd):
        repo_dir = os.path.join(wd, dname)
        if not os.path.isdir(repo_dir) or not os.path.isdir(
            os.path.join(repo_dir, ".git")
        ):
            continue
        __print_action(args.sub_cmd, dname)

        with AdUtil.chdir(repo_dir):
            __do_action(git_helper, args.sub_cmd, extra_args)


def __do_req(url, headers, method="POST", data=dict()):
    if method not in ["POST", "PUT", "DELETE"]:
        raise RuntimeError(f"{method} is unsupported: POST, PUT, DELETE")

    ok = False
    resp_json = None

    if data:
        resp_json = AdUtil.request_json(url, headers=headers, method=method, json=data)
    else:
        resp_json = AdUtil.request_json(url, headers=headers, method=method)

    return (ok, resp_json)


def __rbranch(git_helper, args, extra_args):
    host = os.getenv("MY_GITLAB_HOST", "")
    token = os.getenv("MY_GITLAB_TOKEN", "")

    if not host or not token:
        E("Please make sure you have defined the correct environment variables: {}, {}.".format(crayons.yellow('MY_GITLAB_HOST'), crayons.yellow('MY_GITLAB_TOKEN')))
        return

    sub_cmd = args.sub_cmd
    api_prefix = f"{host}/api/v4"
    headers = {"PRIVATE-TOKEN": token}

    wd = os.path.abspath(".")
    print("{}: {}".format(crayons.yellow("Current workspace"), crayons.green(wd)))

    for dname in os.listdir(wd):
        repo_dir = os.path.join(wd, dname)
        if not os.path.isdir(repo_dir) or not os.path.isdir(
            os.path.join(repo_dir, ".git")
        ):
            continue

        with AdUtil.chdir(repo_dir):
            git_url = git_helper.url()
            pid = urlparse(git_url).path.split(":")[-1]
            # Remove the tailed .git
            if pid.endswith(".git"):
                pid = pid[:-4]

            url = "{}/projects/{}".format(api_prefix, quote_plus(pid))

            if not args.dry_run:
                # Set default branch to current branch
                rok, _ = __do_req(url, headers, "PUT", data={"default_branch": args.name})

                print("Set default branch to {} for project ".format(crayons.yellow(args.name)), end="")

                if rok == 0:
                    print(crayons.green(dname))
                else:
                    print(crayons.red(dname))

                sys.stdout.flush()
            else:
                drmsg = 'curl -X PUT --header "PRIVATE-TOKEN: {}" --header "Content-Type: application/json" --url "{}" --data "{{\\"default_branch\\": \\"{}\\"}}"'.format(
                    token, url, args.name
                )
                print(drmsg)


GIT_CMDS = [
    ("add", "add file contents to index", __common_git_cmd),
    ("checkout", "checkout branch or paths to working tree", __common_git_cmd),
    ("clean", "remove untracked files from working tree", __common_git_cmd),
    ("commit", "record changes to repository", __common_git_cmd),
    ("fetch", "download objects and refs from another repository", __common_git_cmd),
    ("log", "show commit logs", __common_git_cmd),
    (
        "ls-files",
        "information about files in index/working directory",
        __common_git_cmd,
    ),
    ("ls-remote", "show references in a remote repository", __common_git_cmd),
    ("ls-tree", "list contents of a tree object", __common_git_cmd),
    (
        "prune",
        "prune all unreachable objects from the object database",
        __common_git_cmd,
    ),
    (
        "pull",
        "fetch from and merge with another repository or local branch",
        __common_git_cmd,
    ),
    ("push", "update remote refs along with associated objects", __common_git_cmd),
    (
        "rebase",
        "forward-port local commits to the updated upstream head",
        __common_git_cmd,
    ),
    ("merge", "Join two or more development histories together", __common_git_cmd),
    ("remote", "manage set of tracked repositories", __common_git_cmd),
    ("reset", "reset current HEAD to specified state", __common_git_cmd),
    ("revert", "revert existing commits", __common_git_cmd),
    ("stash", "stash away changes to dirty working directory", __common_git_cmd),
    ("status", "show working-tree status", __common_git_cmd),
    ("submodule", "initialize, update, or inspect submodules", __common_git_cmd),
    ("branch", "list, create, or delete branches", __common_git_cmd),
    (
        "tag",
        "create, list, delete or verify tag object signed with GPG",
        __common_git_cmd,
    ),
]


def __create_argparser():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="adgit <sub_cmd> <arguments>",
    )

    parser.add_argument(
        "-v", "--version", action="version", version="{}".format(__version__)
    )
    parser.add_argument(
        "--verbose",
        default=False,
        action="store_true",
        help=textwrap.dedent("""Show verbose information."""),
    )

    parent_parser = [argparse.ArgumentParser(add_help=False)]

    subparsers = parser.add_subparsers(dest="sub_cmd", help="Sub-command help")

    for c in GIT_CMDS:
        sub_parser = subparsers.add_parser(c[0], help=c[1], parents=parent_parser)
        sub_parser.set_defaults(func=c[2])
        sub_parser.add_argument(
            "--dry-run",
            default=False,
            action="store_true",
            help=textwrap.dedent("""Only output command, never do operation."""),
        )

    # rbranch sub-command
    sub_parser = subparsers.add_parser('rbranch', help='Set default branch for remote GitLab repository', parents=parent_parser)
    sub_parser.set_defaults(func=__rbranch)
    sub_parser.add_argument(
        "name", help=textwrap.dedent("rbranch name.")
    )
    sub_parser.add_argument(
        "--dry-run",
        default=False,
        action="store_true",
        help=textwrap.dedent("""Only output command, never do operation."""),
    )

    # lblob sub-command
    sub_parser = subparsers.add_parser('lblob', help='List blob size of each commit', parents=parent_parser)
    sub_parser.set_defaults(func=__list_all_blobs)
    sub_parser.add_argument(
        "repo", help=textwrap.dedent("Repository directory.")
    )
    sub_parser.add_argument(
        "-l", "--limit", type=int,
        default=10 * 1024 * 1024, help=textwrap.dedent("The upper limit, default: 10MB.")
    )
    sub_parser.add_argument(
        "-g", "--gc",
        default=False,
        action="store_true",
        help=textwrap.dedent("Do GC for this repo, DANGEROUS, default: false")
    )
    sub_parser.add_argument(
        "--dry-run",
        default=False,
        action="store_true",
        help=textwrap.dedent("""Only output command, never do operation."""),
    )

    return parser


def __list_all_blobs(gh, args, extra_args):
    git_repo_path = os.path.abspath(args.repo)
    limit = args.limit
    gc = args.gc

    # Check git-filter-repo
    if gc:
        if not AdUtil.command_exists("git-filter-repo"):
            W("Please install git-filter-repo for gc operation: pip install git-filter-repo")
            gc = False

    with AdUtil.chdir(git_repo_path):
        all_commits = []
        code, raw_all_commits = gh.rev_list(__all=None, __pretty_='oneline')

        if code != 0:
            E(raw_all_commits)

        raw_all_commits = raw_all_commits.split('\n')

        for commit in raw_all_commits:
            commit = commit.strip()
            if not commit:
                continue;
            items = commit.split(maxsplit=2)
            if len(items) < 1:
                continue
            commit = items[0]
            all_commits.append(commit)

        commits = {}

        fs = []
        logs = []

        # Parse commits one by one
        for commit in all_commits:
            commits[commit] = []

            code, content = gh.diff_tree(commit, _r=None, _c=None, _M=None, _C=None, __no_commit_id=None)
            content = content.strip()
            if code != 0 or not content:
                continue

            objs = content.split('\n')
            for obj in objs:
                items = obj.split()
                cid = items[3]
                path = items[5]

                # Deleted item
                if cid == "0000000000000000000000000000000000000000":
                    cid = items[2]

                code, size = gh.cat_file(cid, _s=None)
                if code != 0:
                    continue
                size = int(size.strip())
                commits[commit].append({'path': path, 'id': cid, 'size': size})

                if size > limit:
                    fs.append(path.strip())
                    logs.append(f"{commit}, {path}, {size}")

        large_file_info_path = '.{}'.format(str(uuid.uuid4()))
        open(large_file_info_path, 'w+').write("\n".join(logs))

        if fs:
            fs = list(set(fs))
            cmd = "git filter-repo --force --invert-paths --path {}".format(" --path ".join(fs))
            if gc:
                code, _ = AdUtil.run_command(cmd)
                if code != 0:
                    E("Failed to do GC")
            else:
                print(cmd)


def main():
    parser = __create_argparser()

    args = sys.argv[1:]
    if len(args) == 0:
        parser.print_help()
        return

    args, extra_args = parser.parse_known_args()
    sub_cmd = args.sub_cmd

    if sub_cmd != "rbranch":
        git_helper = AdGitHelper(dry_run=args.dry_run)
    else:
        git_helper = AdGitHelper()

    # Always replace --help with -h for git sub-command
    if sub_cmd:
        if "--help" in extra_args or "-h" in extra_args:
            _, ret = git_helper.command(sub_cmd + " -h")
            # Ignore the error code of help
            print(ret)
            return

        args.func(git_helper, args, extra_args)


if __name__ == "__main__":
    with AdUtil.time_consumed():
        main()