[to #42362932] feat: docker_support

* add dockerfile
* uninstall opencv-python-headless
* update develop doc
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9020302

Makefile.docker

@@ -0,0 +1,66 @@
+DOCKER_REGISTRY           = registry.cn-shanghai.aliyuncs.com
+DOCKER_ORG                = modelscope
+DOCKER_IMAGE              = modelscope
+DOCKER_FULL_NAME          = $(DOCKER_REGISTRY)/$(DOCKER_ORG)/$(DOCKER_IMAGE)
+
+# CUDA_VERSION              = 11.3
+# CUDNN_VERSION             = 8
+BASE_RUNTIME              = reg.docker.alibaba-inc.com/pai-dlc/pytorch-training:1.10PAI-gpu-py36-cu113-ubuntu18.04
+BASE_DEVEL                = reg.docker.alibaba-inc.com/pai-dlc/pytorch-training:1.10PAI-gpu-py36-cu113-ubuntu18.04
+
+
+MODELSCOPE_VERSION           = $(shell git describe --tags --always)
+
+# Can be either official / dev
+BUILD_TYPE                = dev
+BUILD_PROGRESS            = auto
+BUILD_ARGS                = --build-arg BASE_IMAGE=$(BASE_IMAGE)
+
+EXTRA_DOCKER_BUILD_FLAGS ?= --network=host
+# DOCKER_BUILD              = DOCKER_BUILDKIT=1 \
+# 							docker build \
+# 								--progress=$(BUILD_PROGRESS) \
+# 								$(EXTRA_DOCKER_BUILD_FLAGS) \
+# 								--target $(BUILD_TYPE) \
+# 								-t $(DOCKER_FULL_NAME):$(DOCKER_TAG) \
+# 								$(BUILD_ARGS) \
+#								-f docker/pytorch.dockerfile .
+DOCKER_BUILD              = DOCKER_BUILDKIT=1 \
+							docker build \
+								$(EXTRA_DOCKER_BUILD_FLAGS) \
+								-t $(DOCKER_FULL_NAME):$(DOCKER_TAG) \
+								$(BUILD_ARGS)  \
+								-f docker/pytorch.dockerfile .
+DOCKER_PUSH               = docker push $(DOCKER_FULL_NAME):$(DOCKER_TAG)
+
+.PHONY: all
+all: devel-image
+
+.PHONY: devel-image
+devel-image: BASE_IMAGE := $(BASE_DEVEL)
+devel-image: DOCKER_TAG := $(MODELSCOPE_VERSION)-devel
+devel-image:
+	$(DOCKER_BUILD)
+
+.PHONY: devel-push
+devel-push: BASE_IMAGE := $(BASE_DEVEL)
+devel-push: DOCKER_TAG := $(MODELSCOPE_VERSION)-devel
+devel-push:
+	$(DOCKER_PUSH)
+
+.PHONY: runtime-image
+runtime-image: BASE_IMAGE := $(BASE_RUNTIME)
+runtime-image: DOCKER_TAG := $(MODELSCOPE_VERSION)-runtime
+runtime-image:
+	$(DOCKER_BUILD)
+	docker tag $(DOCKER_FULL_NAME):$(DOCKER_TAG) $(DOCKER_FULL_NAME):latest
+
+.PHONY: runtime-push
+runtime-push: BASE_IMAGE := $(BASE_RUNTIME)
+runtime-push: DOCKER_TAG := $(MODELSCOPE_VERSION)-runtime
+runtime-push:
+	$(DOCKER_PUSH)
+
+.PHONY: clean
+clean:
+	-docker rmi -f $(shell docker images -q $(DOCKER_FULL_NAME))

docker/.dockerignore

@@ -0,0 +1,4 @@
+*.sh
+*.md
+*.dockerfile
+*.zip

docker/pytorch.dockerfile

@@ -0,0 +1,37 @@
+# syntax = docker/dockerfile:experimental
+#
+# NOTE: To build this you will need a docker version > 18.06 with
+#       experimental enabled and DOCKER_BUILDKIT=1
+#
+#       If you do not use buildkit you are not going to have a good time
+#
+#       For reference:
+#           https://docs.docker.com/develop/develop-images/build_enhancements/
+
+#ARG BASE_IMAGE=reg.docker.alibaba-inc.com/pai-dlc/pytorch-training:1.10PAI-gpu-py36-cu113-ubuntu18.04
+#FROM ${BASE_IMAGE} as dev-base
+
+FROM reg.docker.alibaba-inc.com/pai-dlc/pytorch-training:1.10PAI-gpu-py36-cu113-ubuntu18.04 as dev-base
+# config pip source
+RUN mkdir /root/.pip
+COPY docker/rcfiles/pip.conf.tsinghua  /root/.pip/pip.conf
+
+# install modelscope and its python env
+WORKDIR /opt/modelscope
+COPY . .
+RUN pip install -r requirements.txt
+# RUN --mount=type=cache,target=/opt/ccache \
+#     python setup.py install
+
+# opencv-python-headless conflict with opencv-python installed
+RUN python setup.py install \
+    && pip uninstall -y opencv-python-headless
+
+# prepare modelscope libs
+COPY docker/scripts/install_libs.sh /tmp/
+RUN bash /tmp/install_libs.sh && \
+    rm -rf /tmp/install_libs.sh
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/modelscope/lib64
+
+WORKDIR /workspace

docker/rcfiles/pip.conf.tsinghua

@@ -0,0 +1,2 @@
+[global]
+index-url=https://pypi.tuna.tsinghua.edu.cn/simple

docker/rcfiles/sources.list.aliyun

@@ -0,0 +1,25 @@
+deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted
+# deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted
+
+deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted
+# deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted
+
+deb http://mirrors.aliyun.com/ubuntu/ bionic universe
+# deb-src http://mirrors.aliyun.com/ubuntu/ bionic universe
+deb http://mirrors.aliyun.com/ubuntu/ bionic-updates universe
+# deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates universe
+
+deb http://mirrors.aliyun.com/ubuntu/ bionic multiverse
+# deb-src http://mirrors.aliyun.com/ubuntu/ bionic multiverse
+deb http://mirrors.aliyun.com/ubuntu/ bionic-updates multiverse
+# deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates multiverse
+
+deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
+# deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
+
+deb http://mirrors.aliyun.com/ubuntu bionic-security main restricted
+# deb-src http://mirrors.aliyun.com/ubuntu bionic-security main restricted
+deb http://mirrors.aliyun.com/ubuntu bionic-security universe
+# deb-src http://mirrors.aliyun.com/ubuntu bionic-security universe
+deb http://mirrors.aliyun.com/ubuntu bionic-security multiverse
+# deb-src http://mirrors.aliyun.com/ubuntu bionic-security multiverse

docker/rcfiles/user.vimrc

@@ -0,0 +1,10 @@
+set nocompatible
+set encoding=utf-8
+set hlsearch
+set smartindent
+set ruler
+set number
+set ts=2
+set sw=2
+set expandtab
+autocmd FileType make setlocal noexpandtab

docker/scripts/install_libs.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eo pipefail
+
+ModelScopeLib=/usr/local/modelscope/lib64
+
+if [ ! -d /usr/local/modelscope ]; then
+    mkdir -p $ModelScopeLib
+fi
+
+# audio libs
+wget "http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/libs/audio/libmitaec_pyio.so" -O ${ModelScopeLib}/libmitaec_pyio.so

docs/source/develop.md

@@ -93,3 +93,22 @@ TODO
 ```bash
 make whl
 ```
+
+## Build docker
+
+build develop docker
+```bash
+sudo make -f Makefile.docker devel-image
+```
+
+push develop docker, passwd pls ask wenmeng.zwm
+```bash
+sudo docker login --username=mass_test@test.aliyunid.com registry.cn-shanghai.aliyuncs.com
+Password:
+sudo make -f Makefile.docker devel-push
+```
+
+To build runtime image, just replace `devel` with `runtime` in the upper commands.
+```bash
+udo make -f Makefile.docker runtime-image runtime-push
+```