diff --git a/.codacy.yaml b/.codacy.yaml
new file mode 100644
index 0000000..14735f0
--- /dev/null
+++ b/.codacy.yaml
@@ -0,0 +1,9 @@
+# https://support.codacy.com/hc/en-us/articles/115002130625-Codacy-Configuration-File
+---
+engines:
+ bandit:
+ enabled: false # FIXME: make it work
+exclude_paths:
+- scripts/*
+- setup.py
+- docker/**/*
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..6b8710a
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1 @@
+.git
diff --git a/.gitignore b/.gitignore
index 94ca90d..4a6f60f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,87 +1,131 @@
-# ---> Android
-# Built application files
-*.apk
-*.aar
-*.ap_
-*.aab
-
-# Files for the ART/Dalvik VM
-*.dex
-
-# Java class files
-*.class
-
-# Generated files
-bin/
-gen/
-out/
-# Uncomment the following line in case you need and you don't have the release build type files in your app
-# release/
-
-# Gradle files
-.gradle/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+*~
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
-# Local configuration file (sdk path, etc)
-local.properties
+# Flask stuff:
+instance/
+.webassets-cache
-# Proguard folder generated by Eclipse
-proguard/
+# Scrapy stuff:
+.scrapy
-# Log Files
-*.log
+# Sphinx documentation
+docs/_build/
+docs/test_build/
+docs/build_test/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+venv_/
+venv2/
+venv3/
+venv_doc/
+venv_py2/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+
+# IDE Specific directories
+.DS_Store
+.idea
+.vscode/
+
+# TensorLayer Directories
+checkpoints
+data/
+lib_win/
-# Android Studio Navigation editor temp files
-.navigation/
-
-# Android Studio captures folder
-captures/
-
-# IntelliJ
-*.iml
-.idea/workspace.xml
-.idea/tasks.xml
-.idea/gradle.xml
-.idea/assetWizardSettings.xml
-.idea/dictionaries
-.idea/libraries
-# Android Studio 3 in .gitignore file.
-.idea/caches
-.idea/modules.xml
-# Comment next line if keeping position of elements in Navigation Editor is relevant for you
-.idea/navEditor.xml
-
-# Keystore files
-# Uncomment the following lines if you do not want to check your keystore files in.
-#*.jks
-#*.keystore
-
-# External native build folder generated in Android Studio 2.2 and later
-.externalNativeBuild
-.cxx/
-
-# Google Services (e.g. APIs or Firebase)
-# google-services.json
-
-# Freeline
-freeline.py
-freeline/
-freeline_project_description.json
-
-# fastlane
-fastlane/report.xml
-fastlane/Preview.html
-fastlane/screenshots
-fastlane/test_output
-fastlane/readme.md
-
-# Version control
-vcs.xml
-
-# lint
-lint/intermediates/
-lint/generated/
-lint/outputs/
-lint/tmp/
-# lint/reports/
+# Custom Scripts
+update_tl.bat
+update_tl.py
+# Data Files and ByteCode files
+*.gz
+*.npz
diff --git a/.pyup.yml b/.pyup.yml
new file mode 100644
index 0000000..6c96920
--- /dev/null
+++ b/.pyup.yml
@@ -0,0 +1,75 @@
+############################################################################
+# see https://pyup.io/docs/configuration/ for all available options #
+############################################################################
+
+# configure updates globally
+# default: all
+# allowed: all, insecure, False
+update: all
+
+# configure dependency pinning globally
+# default: True
+# allowed: True, False
+pin: False
+
+# set the default branch
+# default: empty, the default branch on GitHub
+branch: master
+
+# update schedule
+# default: empty
+# allowed: "every day", "every week", ..
+schedule: "every day"
+
+# search for requirement files
+# default: True
+# allowed: True, False
+search: False
+
+# Specify requirement files by hand, default is empty
+# default: empty
+# allowed: list
+requirements:
+ # Requirements for the library
+ - requirements/requirements.txt
+
+ # Requirements for the development
+ - requirements/requirements_tf_cpu.txt
+
+ # Requirements for the development
+ - requirements/requirements_tf_gpu.txt
+
+ # Not necessary, but recommended libraries
+ - requirements/requirements_extra.txt
+
+ # Requirements for contrib loggers
+ - requirements_contrib_loggers.txt
+
+ # Requirements for the db
+ - requirements/requirements_db.txt
+
+ # Requirements for the development
+ - requirements/requirements_dev.txt
+
+ # Requirements for building docs
+ - requirements/requirements_doc.txt
+
+ # Requirements for running unittests
+ - requirements/requirements_test.txt
+
+# add a label to pull requests, default is not set
+# requires private repo permissions, even on public repos
+# default: empty
+#label_prs: update
+
+# configure the branch prefix the bot is using
+# default: pyup-
+branch_prefix: pyup-
+
+# set a global prefix for PRs
+# default: empty
+pr_prefix: "PyUP - Dependency Update"
+
+# allow to close stale PRs
+# default: True
+close_prs: True
diff --git a/.readthedocs.yml b/.readthedocs.yml
new file mode 100644
index 0000000..0752675
--- /dev/null
+++ b/.readthedocs.yml
@@ -0,0 +1,14 @@
+# https://docs.readthedocs.io/en/latest/yaml-config.html
+
+build:
+ image: latest # For python 3.6
+
+formats:
+ - epub
+ - pdf
+
+python:
+ version: 3.6
+
+requirements_file:
+ requirements/requirements_doc.txt
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..63cda3d
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,119 @@
+# https://docs.travis-ci.com/user/languages/python/
+language: python
+
+# https://docs.travis-ci.com/user/caching/#pip-cache
+cache:
+ directories:
+ - $HOME/.cache/pip/wheels
+
+addons:
+ apt:
+ update: false
+
+branches:
+ only:
+ - master
+ - TensorLayer-2.x
+ - /^\d+\.\d+(\.\d+)?(\S*)?$/
+
+python:
+ - "3.6"
+ - "3.5"
+# - "2.7" # TensorLayer 2.0 does not support python2 now
+
+env:
+
+ # Backward Compatibility in insured for release less than 1 year old.
+ # https://pypi.org/project/tensorflow/#history
+ matrix:
+ - _TF_VERSION=2.0.0-rc1
+# - _TF_VERSION=1.12.0 # Remove on Oct 22, 2019
+# - _TF_VERSION=1.11.0 # Remove on Sep 28, 2019
+# - _TF_VERSION=1.10.1 # Remove on Aug 24, 2019
+# - _TF_VERSION=1.9.0 # Remove on Jul 10, 2019
+# - _TF_VERSION=1.8.0 # Remove on Apr 28, 2019
+# - _TF_VERSION=1.7.1 # Remove on May 08, 2019
+# - _TF_VERSION=1.7.0 # Remove on Mar 29, 2019
+# - _TF_VERSION=1.6.0 # Remove on Mar 01, 2019
+
+ global:
+
+ - PYPI_USER='jonathandekhtiar'
+
+ # See https://docs.travis-ci.com/user/encryption-keys/ for more details about secure keys.
+
+ ### == PYPI_PASSWORD === ###
+ ## To update: travis encrypt PYPI_PASSWORD=################################
+ - secure: "fGIRDjfzzP9DhdDshgh/+bWTZ5Y0jTD4aR+gsT1TyAyc6N4f3RRlx70xZZwYMdQ+XC3no/q4na8UzhhuSM0hCCM1EaQ78WF1c6+FBScf4vYGoYgyJ1am+4gu54JXt+4f0bd+s6jyYBafJALUJp5fqHoxCUXqzjrOqGBBU2+JbL71Aaj8yhQuK0VPPABexsQPQM312Gvzg7hy9dh63J0Q02PqINn+CTcwq3gLH9Oua58zWQ7TaT0cdy/hzAc6Yxy3ajo2W5NU+nKROaaG9W57sa7K/v1dshDFFFST2DdGxm9i7vvfPsq0OWM6qWLsec/4mXJWsmai2ygZEv+IhaABb10c7spd2nl7oHFj2UGmldtO5W0zLb1KkCPWDPilFt3lvHM+OS/YaibquL5/5+yGj0LsRJrVyWoMBA8idcQeH4dvTAfySeFpO42VNwW5ez9JaEOh7bBp7naAA8c/fbNJJ5YEW4MEmOZ9dwFTohNNDiN+oITSrcXBS+jukbfTOmtCeYNUker+4G2YwII9cxHXbZeIMrTq9AqTfOVTAYCFaFHKbpSc1+HCyF7n5ZfNC00kBaw93XUnLRzSNKe5Ir791momYL8HecMN3OAI77bz26/pHSfzJnLntq9qx2nLBTnqDuSq5/pHvdZ8hyt+hTDxvF7HJIVMhbnkjoLPxmn4k/I="
+
+ ### === GITHUB_PERSONAL_TOKEN === ###
+ ## To update: travis encrypt GITHUB_PERSONAL_TOKEN=################################
+ - secure: "kMxg4WfTwhnYMD7WjYk71vgw7XlShPpANauKzfTL6oawDrpQRkBUai4uQwiL3kXVBuVv9rKKKZxxnyAm05iB5wGasPDhlFA1DPF0IbyU3pwQKlw2Xo5qtHdgxBnbms6JJ9z5b+hHCVg+LXYYeUw5qG01Osg5Ue6j1g2juQQHCod00FNuo3fe8ah/Y10Rem6CigH8ofosCrTvN2w1GaetJwVehRYf8JkPC6vQ+Yk8IIjHn2CaVJALbhuchVblxwH0NXXRen915BVBwWRQtsrvEVMXKu7A8sMHmvPz1u3rhXQfjpF2KeVOfy1ZnyiHcLE2HgAPuAGh4kxZAAA8ovmcaCbv8m64bm72BrQApSbt6OEtR9L1UeUwdEgi54FH1XFOHQ9dA6CpiGCyk3wAJZqO0/PkNYVLfb4gPLvnvBRwYBaPgUPvVNhidFu/oODENZmcf8g9ChtuC1GT70EYlVwhgDGqUY7/USZCEvIPe81UToqtIKgcgA+Is51XindumJVMiqTsjgdqeC/wZcw+y37TAjIvvXbtYxeqIKv9zh1JuZppqUhnf+OhI+HHFaY4iu7lQTs3C0WmoLskZAp9srwRtifnVFFkdYzngmPaSjWyko2qiS0cTdFJQB/ljqmnJdksacbv5OOa0Q4qZef/hW774nVx105FlkAIk70D2b5l2pA="
+
+ ### === GITHUB_PERSONAL_TOKEN === ###
+ ## To update: travis encrypt HYPERDASH_APIKEY=################################
+ - secure: "ez9Ck1VpqWOd2PPu+CMWzd8R4aAIXbjKCk96PCKwWu8VXoHjaPkiy8Nn0LUzSlUg3nKdZmu2JSndwDMy3+lMLG7zE2WlGNY7MF5KM3GrvFpP3cxJQ6OuPcZcEH4j5KtBtNTrNqa8SWglqhc9mr66a92SD8Ydc4aMj6L9nbQvrsvVzIMmMy6weVlpBF35nweYCM8LxlsnqyPLleHPZo3o/k+hsTqQQbiMGjC78tqrGr56u7AjL2+D/m33+dfCGzFvMJFcpLQ5ldbcVU54i5e6V3xJ48P30QOGZaqG3fcpfZsyJEIWjykt6XFA8GfJjaVVbxdlr7zP7Vd9iWBuemnMEX3F9Cy/4x7LmX9PJfsVPC6FQnanDvsZSNO5hpmKe8BTpttJJvxgscOczV4jnI69OzqhSQeyChwtkqhIg1E/53XIO+uLJAAZsCkAco7tjGGXTKyv8ZlpSJwSqsLcmgpmQbfodCoMLcYenTxqKZv78e2B4tOPGQyS2bkSxAqhvAIam7RCq/yEvz5n2/mBFEGwP6OQFIdC7ypO2LyrOlLT7HJjCeYMeKSm+GOD3LW9oIy9QJZpG6N/zAAjnk9C2mYtWRQIBo4qdHjRvyDReevDexI8j0AXySblxREmQ7ZaT6KEDXXZSu5goTlaGm0g2HwAkKu9xYFV/bRtp6+i1mP7CQg="
+
+matrix:
+ include:
+ - python: '3.6'
+ env:
+ - _DOC_AND_YAPF_TEST=True
+
+install:
+ - |
+ if [[ -v _DOC_AND_YAPF_TEST ]]; then
+ pip install tensorflow==2.0.0-rc1
+ pip install yapf
+ pip install -e .[doc]
+ else
+ pip install tensorflow==$_TF_VERSION
+ pip install -e .[all_cpu_dev]
+ fi
+
+script:
+ # units test
+ # https://docs.pytest.org/en/latest/
+ - rm setup.cfg
+ - |
+ if [[ -v _DOC_AND_YAPF_TEST ]]; then
+ mv setup.travis_doc.cfg setup.cfg
+ else
+ mv setup.travis.cfg setup.cfg
+ fi
+ - pytest
+
+
+before_deploy:
+ - python setup.py sdist
+ - python setup.py bdist_wheel
+ - python setup.py bdist_wheel --universal
+ - python setup.py egg_info
+
+
+deploy:
+
+# Documentation: https://docs.travis-ci.com/user/deployment/pypi/
+- provider: pypi
+ user: '$PYPI_USER'
+ password: '$PYPI_PASSWORD'
+ skip_cleanup: true
+ on:
+ tags: true
+ python: '3.6'
+ condition: '$_TF_VERSION = 2.0.0-rc1'
+# condition: '$_TF_VERSION = 1.11.0'
+
+# Documentation: https://docs.travis-ci.com/user/deployment/releases/
+- provider: releases
+ file:
+ - dist/*
+ - tensorlayer.egg-info/PKG-INFO
+ file_glob: true
+ skip_cleanup: true
+ api_key: '$GITHUB_PERSONAL_TOKEN'
+ on:
+ tags: true
+ python: '3.6'
+ condition: '$_TF_VERSION = 2.0.0-rc1'
+# condition: '$_TF_VERSION = 1.11.0'
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..a19b722
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,585 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+
+
+
+
+
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Dependencies Update
+
+### Deprecated
+
+### Fixed
+
+- Fix README. (#PR 1044)
+- Fix package info. (#PR 1046)
+
+### Removed
+
+### Security
+
+### Contributors
+
+- @luomai (PR #1044, 1046)
+
+
+## [2.2.0] - 2019-09-13
+
+TensorLayer 2.2.0 is a maintenance release.
+It contains numerous API improvement and bug fixes.
+This release is compatible with TensorFlow 2 RC1.
+
+### Added
+- Support nested layer customization (#PR 1015)
+- Support string dtype in InputLayer (#PR 1017)
+- Support Dynamic RNN in RNN (#PR 1023)
+- Add ResNet50 static model (#PR 1030)
+- Add performance test code in static model (#PR 1041)
+
+### Changed
+
+- `SpatialTransform2dAffine` auto `in_channels`
+- support TensorFlow 2.0.0-rc1
+- Update model weights property, now returns its copy (#PR 1010)
+
+### Fixed
+- RNN updates: remove warnings, fix if seq_len=0, unitest (#PR 1033)
+- BN updates: fix BatchNorm1d for 2D data, refactored (#PR 1040)
+
+### Dependencies Update
+
+### Deprecated
+
+### Fixed
+- Fix `tf.models.Model._construct_graph` for list of outputs, e.g. STN case (PR #1010)
+- Enable better `in_channels` exception raise. (PR #1015)
+- Set allow_pickle=True in np.load() (#PR 1021)
+- Remove `private_method` decorator (#PR 1025)
+- Copy original model's `trainable_weights` and `nontrainable_weights` when initializing `ModelLayer` (#PR 1026)
+- Copy original model's `trainable_weights` and `nontrainable_weights` when initializing `LayerList` (#PR 1029)
+- Remove redundant parts in `model.all_layers` (#PR 1029)
+- Replace `tf.image.resize_image_with_crop_or_pad` with `tf.image.resize_with_crop_or_pad` (#PR 1032)
+- Fix a bug in `ResNet50` static model (#PR 1041)
+
+### Removed
+
+### Security
+
+### Contributors
+
+- @zsdonghao
+- @luomai
+- @ChrisWu1997: #1010 #1015 #1025 #1030 #1040
+- @warshallrho: #1017 #1021 #1026 #1029 #1032 #1041
+- @ArnoldLIULJ: #1023
+- @JingqingZ: #1023
+
+## [2.1.0]
+
+### Changed
+- Add version_info in model.config. (PR #992)
+- Replace tf.nn.func with tf.nn.func.\_\_name\_\_ in model config. (PR #994)
+- Add Reinforcement learning tutorials. (PR #995)
+- Add RNN layers with simple rnn cell, GRU cell, LSTM cell. (PR #998)
+- Update Seq2seq (#998)
+- Add Seq2seqLuongAttention model (#998)
+
+### Fixed
+
+### Contributors
+- @warshallrho: #992 #994
+- @quantumiracle: #995
+- @Tokarev-TT-33: #995
+- @initial-h: #995
+- @Officium: #995
+- @ArnoldLIULJ: #998
+- @JingqingZ: #998
+
+
+## [2.0.2] - 2019-6-5
+
+### Changed
+- change the format of network config, change related code and files; change layer act (PR #980)
+
+### Fixed
+- Fix dynamic model cannot track PRelu weights gradients problem (PR #982)
+- Raise .weights warning (commit)
+
+### Contributors
+- @warshallrho: #980
+- @1FengL: #982
+
+## [2.0.1] - 2019-5-17
+
+
+A maintain release.
+
+### Changed
+- remove `tl.layers.initialize_global_variables(sess)` (PR #931)
+- support `trainable_weights` (PR #966)
+
+### Added
+ - Layer
+ - `InstanceNorm`, `InstanceNorm1d`, `InstanceNorm2d`, `InstanceNorm3d` (PR #963)
+
+* Reinforcement learning tutorials. (PR #995)
+
+### Changed
+- remove `tl.layers.initialize_global_variables(sess)` (PR #931)
+- update `tutorial_generate_text.py`, `tutorial_ptb_lstm.py`. remove `tutorial_ptb_lstm_state_is_tuple.py` (PR #958)
+- change `tl.layers.core`, `tl.models.core` (PR #966)
+- change `weights` into `all_weights`, `trainable_weights`, `nontrainable_weights`
+
+### Dependencies Update
+- nltk>=3.3,<3.4 => nltk>=3.3,<3.5 (PR #892)
+- pytest>=3.6,<3.11 => pytest>=3.6,<4.1 (PR #889)
+- yapf>=0.22,<0.25 => yapf==0.25.0 (PR #896)
+- imageio==2.5.0 progressbar2==3.39.3 scikit-learn==0.21.0 scikit-image==0.15.0 scipy==1.2.1 wrapt==1.11.1 pymongo==3.8.0 sphinx==2.0.1 wrapt==1.11.1 opencv-python==4.1.0.25 requests==2.21.0 tqdm==4.31.1 lxml==4.3.3 pycodestyle==2.5.0 sphinx==2.0.1 yapf==0.27.0(PR #967)
+
+### Fixed
+- fix docs of models @zsdonghao #957
+- In `BatchNorm`, keep dimensions of mean and variance to suit `channels first` (PR #963)
+
+### Contributors
+- @warshallrho: #PR966
+- @zsdonghao: #931
+- @yd-yin: #963
+- @Tokarev-TT-33: # 995
+- @initial-h: # 995
+- @quantumiracle: #995
+- @Officium: #995
+- @1FengL: #958
+- @dvklopfenstein: #971
+
+
+## [2.0.0] - 2019-05-04
+
+To many PR for this update, please check [here](https://github.com/tensorlayer/tensorlayer/releases/tag/2.0.0) for more details.
+
+### Changed
+* update for TensorLayer 2.0.0 alpha version (PR #952)
+* support TensorFlow 2.0.0-alpha
+* support both static and dynamic model building
+
+### Dependencies Update
+- tensorflow>=1.6,<1.13 => tensorflow>=2.0.0-alpha (PR #952)
+- h5py>=2.9 (PR #952)
+- cloudpickle>=0.8.1 (PR #952)
+- remove matplotlib
+
+### Contributors
+- @zsdonghao
+- @JingqingZ
+- @ChrisWu1997
+- @warshallrho
+
+
+## [1.11.1] - 2018-11-15
+
+### Changed
+* guide for pose estimation - flipping (PR #884)
+* cv2 transform support 2 modes (PR #885)
+
+### Dependencies Update
+- pytest>=3.6,<3.9 => pytest>=3.6,<3.10 (PR #874)
+- requests>=2.19,<2.20 => requests>=2.19,<2.21 (PR #874)
+- tqdm>=4.23,<4.28 => tqdm>=4.23,<4.29 (PR #878)
+- pytest>=3.6,<3.10 => pytest>=3.6,<3.11 (PR #886)
+- pytest-xdist>=1.22,<1.24 => pytest-xdist>=1.22,<1.25 (PR #883)
+- tensorflow>=1.6,<1.12 => tensorflow>=1.6,<1.13 (PR #886)
+
+### Contributors
+- @zsdonghao: #884 #885
+
+## [1.11.0] - 2018-10-18
+
+### Added
+- Layer:
+ - Release `GroupNormLayer` (PR #850)
+- Image affine transformation APIs
+ - `affine_rotation_matrix` (PR #857)
+ - `affine_horizontal_flip_matrix` (PR #857)
+ - `affine_vertical_flip_matrix` (PR #857)
+ - `affine_shift_matrix` (PR #857)
+ - `affine_shear_matrix` (PR #857)
+ - `affine_zoom_matrix` (PR #857)
+ - `affine_transform_cv2` (PR #857)
+ - `affine_transform_keypoints` (PR #857)
+- Affine transformation tutorial
+ - `examples/data_process/tutorial_fast_affine_transform.py` (PR #857)
+
+### Changed
+- BatchNormLayer: support `data_format`
+
+### Dependencies Update
+- matplotlib>=2.2,<2.3 => matplotlib>=2.2,<3.1 (PR #845)
+- pydocstyle>=2.1,<2.2 => pydocstyle>=2.1,<3.1 (PR #866)
+- scikit-learn>=0.19,<0.20 => scikit-learn>=0.19,<0.21 (PR #851)
+- sphinx>=1.7,<1.8 => sphinx>=1.7,<1.9 (PR #842)
+- tensorflow>=1.6,<1.11 => tensorflow>=1.6,<1.12 (PR #853)
+- tqdm>=4.23,<4.26 => tqdm>=4.23,<4.28 (PR #862 & #868)
+- yapf>=0.22,<0.24 => yapf>=0.22,<0.25 (PR #829)
+
+### Fixed
+- Correct offset calculation in `tl.prepro.transform_matrix_offset_center` (PR #855)
+
+### Contributors
+- @2wins: #850 #855
+- @DEKHTIARJonathan: #853
+- @zsdonghao: #857
+- @luomai: #857
+
+## [1.10.1] - 2018-09-07
+
+### Added
+- unittest `tests\test_timeout.py` has been added to ensure the network creation process does not freeze.
+
+### Changed
+ - remove 'tensorboard' param, replaced by 'tensorboard_dir' in `tensorlayer/utils.py` with customizable tensorboard directory (PR #819)
+
+### Removed
+- TL Graph API removed. Memory Leaks Issues with this API, will be fixed and integrated in TL 2.0 (PR #818)
+
+### Fixed
+- Issue #817 fixed: TL 1.10.0 - Memory Leaks and very slow network creation.
+
+### Dependencies Update
+- autopep8>=1.3,<1.4 => autopep8>=1.3,<1.5 (PR #815)
+- imageio>=2.3,<2.4 => imageio>=2.3,<2.5 (PR #823)
+- pytest>=3.6,<3.8 => pytest>=3.6,<3.9 (PR #823)
+- pytest-cov>=2.5,<2.6 => pytest-cov>=2.5,<2.7 (PR #820)
+
+### Contributors
+- @DEKHTIARJonathan: #815 #818 #820 #823
+- @ndiy: #819
+- @zsdonghao: #818
+
+
+## [1.10.0] - 2018-09-02
+
+### Added
+- API:
+ - Add `tl.model.vgg19` (PR #698)
+ - Add `tl.logging.contrib.hyperdash` (PR #739)
+ - Add `tl.distributed.trainer` (PR #700)
+ - Add `prefetch_buffer_size` to the `tl.distributed.trainer` (PR #766)
+ - Add `tl.db.TensorHub` (PR ＃751)
+ - Add `tl.files.save_graph` (PR ＃751)
+ - Add `tl.files.load_graph_` (PR ＃751)
+ - Add `tl.files.save_graph_and_params` (PR ＃751)
+ - Add `tl.files.load_graph_and_params` (PR ＃751)
+ - Add `tl.prepro.keypoint_random_xxx` (PR #787)
+- Documentation:
+ - Add binary, ternary and dorefa links (PR #711)
+ - Update input scale of VGG16 and VGG19 to 0~1 (PR #736)
+ - Update database (PR ＃751)
+- Layer:
+ - Release SwitchNormLayer (PR #737)
+ - Release QuanConv2d, QuanConv2dWithBN, QuanDenseLayer, QuanDenseLayerWithBN (PR#735)
+ - Update Core Layer to support graph (PR ＃751)
+ - All Pooling layers support `data_format` (PR #809)
+- Setup:
+ - Creation of installation flaggs `all_dev`, `all_cpu_dev`, and `all_gpu_dev` (PR #739)
+- Examples:
+ - change folder struction (PR #802)
+ - `tutorial_models_vgg19` has been introduced to show how to use `tl.model.vgg19` (PR #698).
+ - fix bug of `tutorial_bipedalwalker_a3c_continuous_action.py` (PR #734, Issue #732)
+ - `tutorial_models_vgg16` and `tutorial_models_vgg19` has been changed the input scale from [0,255] to [0,1](PR #710)
+ - `tutorial_mnist_distributed_trainer.py` and `tutorial_cifar10_distributed_trainer.py` are added to explain the uses of Distributed Trainer (PR #700)
+ - add `tutorial_quanconv_cifar10.py` and `tutorial_quanconv_mnist.py` (PR #735)
+ - add `tutorial_work_with_onnx.py`(PR #775)
+- Applications:
+ - [Arbitrary Style Transfer in Real-time with Adaptive Instance Normalization](https://arxiv.org/abs/1703.06868) (PR #799)
+
+### Changed
+ - function minibatches changed to avoid wasting samples.(PR #762)
+ - all the input scale in both vgg16 and vgg19 has been changed the input scale from [0,255] to [0,1](PR #710)
+ - Dockerfiles merged and refactored into one file (PR #747)
+ - LazyImports move to the most **top level** imports as possible (PR #739)
+ - some new test functions have been added in `test_layers_convolution.py`, `test_layers_normalization.py`, `test_layers_core.py` (PR #735)
+ - documentation now uses mock imports reducing the number of dependencies to compile the documentation (PR #785)
+ - fixed and enforced pydocstyle D210, D200, D301, D207, D403, D204, D412, D402, D300, D208 (PR #784)
+
+### Deprecated
+ - `tl.logging.warn` has been deprecated in favor of `tl.logging.warning` (PR #739)
+
+### Removed
+ - `conv_layers()` has been removed in both vgg16 and vgg19(PR #710)
+ - graph API (PR #818)
+
+### Fixed
+- import error caused by matplotlib on OSX (PR #705)
+- missing import in tl.prepro (PR #712)
+- Dockerfiles import error fixed - issue #733 (PR #747)
+- Fix a typo in `absolute_difference_error` in file: `tensorlayer/cost.py` - Issue #753 (PR #759)
+- Fix the bug of scaling the learning rate of trainer (PR #776)
+- log error instead of info when npz file not found. (PR #812)
+
+### Dependencies Update
+- numpy>=1.14,<1.15 => numpy>=1.14,<1.16 (PR #754)
+- pymongo>=3.6,<3.7 => pymongo>=3.6,<3.8 (PR #750)
+- pytest>=3.6,<3.7 => tqdm>=3.6,<3.8 (PR #798)
+- pytest-xdist>=1.22,<1.23 => pytest-xdist>=1.22,<1.24 (PR #805 and #806)
+- tensorflow>=1.8,<1.9 => tensorflow>=1.6,<1.11 (PR #739 and PR #798)
+- tqdm>=4.23,<4.25 => tqdm>=4.23,<4.26 (PR #798)
+- yapf>=0.21,<0.22 => yapf>=0.22,<0.24 (PR #798 #808)
+
+### Contributors
+- @DEKHTIARJonathan: #739 #747 #750 #754
+- @lgarithm: #705 #700
+- @OwenLiuzZ: #698 #710 #775 #776
+- @zsdonghao: #711 #712 #734 #736 #737 #700 #751 #809 #818
+- @luomai: #700 #751 #766 #802
+- @XJTUWYD: #735
+- @mutewall: #735
+- @thangvubk: #759
+- @JunbinWang: #796
+- @boldjoel: #787
+
+## [1.9.1] - 2018-07-30
+
+### Fixed
+- Issue with tensorflow 1.10.0 fixed
+
+## [1.9.0] - 2018-06-16
+
+### Added
+- API:
+ - `tl.alphas` and `tl.alphas_like` added following the tf.ones/zeros and tf.zeros_like/ones_like (PR #580)
+ - `tl.lazy_imports.LazyImport` to import heavy libraries only when necessary (PR #667)
+ - `tl.act.leaky_relu6` and `tl.layers.PRelu6Layer` have been deprecated (PR #686)
+ - `tl.act.leaky_twice_relu6` and `tl.layers.PTRelu6Layer` have been deprecated (PR #686)
+- CI Tool:
+ - [Stale Probot](https://github.com/probot/stale) added to clean stale issues (PR #573)
+ - [Changelog Probot](https://github.com/mikz/probot-changelog) Configuration added (PR #637)
+ - Travis Builds now handling a matrix of TF Version from TF==1.6.0 to TF==1.8.0 (PR #644)
+ - CircleCI added to build and upload Docker Containers for each PR merged and tag release (PR #648)
+- Decorator:
+ - `tl.decorators` API created including `deprecated_alias` and `private_method` (PR #660)
+ - `tl.decorators` API enriched with `protected_method` (PR #675)
+ - `tl.decorators` API enriched with `deprecated` directly raising warning and modifying documentation (PR #691)
+- Docker:
+ - Containers for each release and for each PR merged on master built (PR #648)
+ - Containers built in the following configurations (PR #648):
+ - py2 + cpu
+ - py2 + gpu
+ - py3 + cpu
+ - py3 + gpu
+- Documentation:
+ - Clean README.md (PR #677)
+ - Release semantic version added on index page (PR #633)
+ - Optimizers page added (PR #636)
+ - `AMSGrad` added on Optimizers page added (PR #636)
+- Layer:
+ - ElementwiseLambdaLayer added to use custom function to connect multiple layer inputs (PR #579)
+ - AtrousDeConv2dLayer added (PR #662)
+ - Fix bugs of using `tf.layers` in CNN (PR #686)
+- Optimizer:
+
+ - AMSGrad Optimizer added based on `On the Convergence of Adam and Beyond (ICLR 2018)` (PR #636)
+- Setup:
+
+ - Creation of installation flaggs `all`, `all_cpu`, and `all_gpu` (PR #660)
+- Test:
+ - `test_utils_predict.py` added to reproduce and fix issue #288 (PR #566)
+ - `Layer_DeformableConvolution_Test` added to reproduce issue #572 with deformable convolution (PR #573)
+ - `Array_Op_Alphas_Test` and `Array_Op_Alphas_Like_Test` added to test `tensorlayer/array_ops.py` file (PR #580)
+ - `test_optimizer_amsgrad.py` added to test `AMSGrad` optimizer (PR #636)
+ - `test_logging.py` added to insure robustness of the logging API (PR #645)
+ - `test_decorators.py` added (PR #660)
+ - `test_activations.py` added (PR #686)
+- Tutorials:
+ - `tutorial_tfslim` has been introduced to show how to use `SlimNetsLayer` (PR #560).
+ - add the following to all tutorials (PR #697):
+ ```python
+ tf.logging.set_verbosity(tf.logging.DEBUG)
+ tl.logging.set_verbosity(tl.logging.DEBUG)
+ ```
+
+### Changed
+- Tensorflow CPU & GPU dependencies moved to separated requirement files in order to allow PyUP.io to parse them (PR #573)
+- The document of LambdaLayer for linking it with ElementwiseLambdaLayer (PR #587)
+- RTD links point to stable documentation instead of latest used for development (PR #633)
+- TF Version older than 1.6.0 are officially unsupported and raises an exception (PR #644)
+- README.md Badges Updated with Support Python and Tensorflow Versions (PR #644)
+- TL logging API has been consistent with TF logging API and thread-safe (PR #645)
+- Relative Imports changed for absolute imports (PR #657)
+- `tl.files` refactored into a directory with numerous files (PR #657)
+- `tl.files.voc_dataset` fixed because of original Pascal VOC website was down (PR #657)
+- extra requirements hidden inside the library added in the project requirements (PR #657)
+- requirements files refactored in `requirements/` directory (PR #657)
+- README.md and other markdown files have been refactored and cleaned. (PR #639)
+- Ternary Convolution Layer added in unittest (PR #658)
+- Convolution Layers unittests have been cleaned & refactored (PR #658)
+- All the tests are now using a DEBUG level verbosity when run individualy (PR #660)
+- `tf.identity` as activation is **ignored**, thus reducing the size of the graph by removing useless operation (PR #667)
+- argument dictionaries are now checked and saved within the `Layer` Base Class (PR #667)
+- `Layer` Base Class now presenting methods to update faultlessly `all_layers`, `all_params`, and `all_drop` (PR #675)
+- Input Layers have been removed from `tl.layers.core` and added to `tl.layers.inputs` (PR #675)
+- Input Layers are now considered as true layers in the graph (they represent a placeholder), unittests have been updated (PR #675)
+- Layer API is simplified, with automatic feeding `prev_layer` into `self.inputs` (PR #675)
+- Complete Documentation Refactoring and Reorganization (namely Layer APIs) (PR #691)
+
+### Deprecated
+- `tl.layers.TimeDistributedLayer` argurment `args` is deprecated in favor of `layer_args` (PR #667)
+- `tl.act.leaky_relu` have been deprecated in favor of `tf.nn.leaky_relu` (PR #686)
+
+### Removed
+- `assert()` calls remove and replaced by `raise AssertionError()` (PR #667)
+- `tl.identity` is removed, not used anymore and deprecated for a long time (PR #667)
+- All Code specific to `TF.__version__ < "1.6"` have been removed (PR #675)
+
+### Fixed
+- Issue #498 - Deprecation Warning Fix in `tl.layers.RNNLayer` with `inspect` (PR #574)
+- Issue #498 - Deprecation Warning Fix in `tl.files` with truth value of an empty array is ambiguous (PR #575)
+- Issue #565 related to `tl.utils.predict` fixed - `np.hstack` problem in which the results for multiple batches are stacked along `axis=1` (PR #566)
+- Issue #572 with `tl.layers.DeformableConv2d` fixed (PR #573)
+- Issue #664 with `tl.layers.ConvLSTMLayer` fixed (PR #676)
+- Typo of the document of ElementwiseLambdaLayer (PR #588)
+- Error in `tl.layers.TernaryConv2d` fixed - self.inputs not defined (PR #658)
+- Deprecation warning fixed in `tl.layers.binary._compute_threshold()` (PR #658)
+- All references to `tf.logging` replaced by `tl.logging` (PR #661)
+- Duplicated code removed when bias was used (PR #667)
+- `tensorlayer.third_party.roi_pooling.roi_pooling.roi_pooling_ops` is now lazy loaded to prevent systematic error raised (PR #675)
+- Documentation not build in RTD due to old version of theme in docs directory fixed (PR #703)
+- Tutorial:
+ - `tutorial_word2vec_basic.py` saving issue #476 fixed (PR #635)
+ - All tutorials tested and errors have been fixed (PR #635)
+
+### Dependencies Update
+- Update pytest from 3.5.1 to 3.6.0 (PR #647)
+- Update progressbar2 from 3.37.1 to 3.38.0 (PR #651)
+- Update scikit-image from 0.13.1 to 0.14.0 (PR #656)
+- Update keras from 2.1.6 to 2.2.0 (PR #684)
+- Update requests from 2.18.4 to 2.19.0 (PR #695)
+
+### Contributors
+- @lgarithm: #563
+- @DEKHTIARJonathan: #573 #574 #575 #580 #633 #635 #636 #639 #644 #645 #648 #657 #667 #658 #659 #660 #661 #666 #667 #672 #675 #683 #686 #687 #690 #691 #692 #703
+- @2wins: #560 #566 #662
+- @One-sixth: #579
+- @zsdonghao: #587 #588 #639 #685 #697
+- @luomai: #639 #677
+- @dengyueyun666: #676
+
+## [1.8.5] - 2018-05-09
+
+### Added
+- Github Templates added (by @DEKHTIARJonathan)
+ - New issues Template
+ - New PR Template
+- Travis Deploy Automation on new Tag (by @DEKHTIARJonathan)
+ - Deploy to PyPI and create a new version.
+ - Deploy to Github Releases and upload the wheel files
+- PyUP.io has been added to ensure we are compatible with the latest libraries (by @DEKHTIARJonathan)
+- `deconv2d` now handling dilation_rate (by @zsdonghao)
+- Documentation unittest added (by @DEKHTIARJonathan)
+- `test_layers_core` has been added to ensure that `LayersConfig` is abstract.
+
+### Changed
+- All Tests Refactored - Now using unittests and runned with PyTest (by @DEKHTIARJonathan)
+- Documentation updated (by @zsdonghao)
+- Package Setup Refactored (by @DEKHTIARJonathan)
+- Dataset Downlaod now using library progressbar2 (by @DEKHTIARJonathan)
+- `deconv2d` function transformed into Class (by @zsdonghao)
+- `conv1d` function transformed into Class (by @zsdonghao)
+- super resolution functions transformed into Class (by @zsdonghao)
+- YAPF coding style improved and enforced (by @DEKHTIARJonathan)
+
+### Fixed
+- Backward Compatibility Restored with deprecation warnings (by @DEKHTIARJonathan)
+- Tensorflow Deprecation Fix (Issue #498):
+ - AverageEmbeddingInputlayer (by @zsdonghao)
+ - load_mpii_pose_dataset (by @zsdonghao)
+- maxPool2D initializer issue #551 (by @zsdonghao)
+- `LayersConfig` class has been enforced as abstract
+- Pooling Layer Issue #557 fixed (by @zsdonghao)
+
+### Dependencies Update
+- scipy>=1.0,<1.1 => scipy>=1.1,<1.2
+
+### Contributors
+@zsdonghao @luomai @DEKHTIARJonathan
+
+[Unreleased]: https://github.com/tensorlayer/tensorlayer/compare/2.0....master
+[2.2.0]: https://github.com/tensorlayer/tensorlayer/compare/2.2.0...2.2.0
+[2.1.0]: https://github.com/tensorlayer/tensorlayer/compare/2.1.0...2.1.0
+[2.0.2]: https://github.com/tensorlayer/tensorlayer/compare/2.0.2...2.0.2
+[2.0.1]: https://github.com/tensorlayer/tensorlayer/compare/2.0.1...2.0.1
+[2.0.0]: https://github.com/tensorlayer/tensorlayer/compare/2.0.0...2.0.0
+[1.11.1]: https://github.com/tensorlayer/tensorlayer/compare/1.11.0...1.11.0
+[1.11.0]: https://github.com/tensorlayer/tensorlayer/compare/1.10.1...1.11.0
+[1.10.1]: https://github.com/tensorlayer/tensorlayer/compare/1.10.0...1.10.1
+[1.10.0]: https://github.com/tensorlayer/tensorlayer/compare/1.9.1...1.10.0
+[1.9.1]: https://github.com/tensorlayer/tensorlayer/compare/1.9.0...1.9.1
+[1.9.0]: https://github.com/tensorlayer/tensorlayer/compare/1.8.5...1.9.0
+[1.8.5]: https://github.com/tensorlayer/tensorlayer/compare/1.8.4...1.8.5
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..6a010a2
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,199 @@
+# TensorLayer Contributor Guideline
+
+## Welcome to contribute!
+You are more than welcome to contribute to TensorLayer! If you have any improvement, please send us your [pull requests](https://help.github.com/en/articles/about-pull-requests). You may implement your improvement on your [fork](https://help.github.com/en/articles/working-with-forks).
+
+## Checklist
+* Continuous integration
+* Build from sources
+* Unittest
+* Documentation
+* General intro to TensorLayer2
+* How to contribute a new `Layer`
+* How to contribute a new `Model`
+* How to contribute a new example/tutorial
+
+## Continuous integration
+
+We appreciate contributions
+either by adding / improving examples or extending / fixing the core library.
+To make your contributions, you would need to follow the [pep8](https://www.python.org/dev/peps/pep-0008/) coding style and [numpydoc](https://numpydoc.readthedocs.io/en/latest/) document style.
+We rely on Continuous Integration (CI) for checking push commits.
+The following tools are used to ensure that your commits can pass through the CI test:
+
+* [yapf](https://github.com/google/yapf) (format code), compulsory
+* [isort](https://github.com/timothycrosley/isort) (sort imports), optional
+* [autoflake](https://github.com/myint/autoflake) (remove unused imports), optional
+
+You can simply run
+
+```bash
+make format
+```
+
+to apply those tools before submitting your PR.
+
+## Build from sources
+
+```bash
+# First clone the repository and change the current directory to the newly cloned repository
+git clone https://github.com/zsdonghao/tensorlayer2.git
+cd tensorlayer2
+
+# Install virtualenv if necessary
+pip install virtualenv
+
+# Then create a virtualenv called `venv`
+virtualenv venv
+
+# Activate the virtualenv
+
+## Linux:
+source venv/bin/activate
+
+## Windows:
+venv\Scripts\activate.bat
+
+# ============= IF TENSORFLOW IS NOT ALREADY INSTALLED ============= #
+
+# basic installation
+pip install .
+
+# advanced: for a machine **without** an NVIDIA GPU
+pip install -e ".[all_cpu_dev]"
+
+# advanced: for a machine **with** an NVIDIA GPU
+pip install -e ".[all_gpu_dev]"
+```
+
+## Unittest
+
+Launching the unittest for the whole repo:
+
+```bash
+# install pytest
+pip install pytest
+
+# run pytest
+pytest
+```
+
+Running your unittest code on your implemented module only:
+
+```bash
+# install coverage
+pip install coverage
+
+cd /path/to/your/unittest/code
+# For example: cd tests/layers/
+
+# run unittest
+coverage run --source myproject.module -m unittest discover
+# For example: coverage run --source tensorlayer.layers -m unittest discover
+
+# generate html report
+coverage html
+```
+
+## Documentation
+Even though you follow [numpydoc](https://numpydoc.readthedocs.io/en/latest/) document style when writing your code,
+this does not ensure those lines appear on TensorLayer online documentation.
+You need further modify corresponding RST files in `docs/modules`.
+
+For example, to add your implemented new pooling layer into documentation, modify `docs/modules/layer.rst`. First, insert layer name under Layer list
+```rst
+Layer list
+----------
+
+.. autosummary::
+
+ NewPoolingLayer
+```
+
+Second, find pooling layer part and add:
+```rst
+.. -----------------------------------------------------------
+.. Pooling Layers
+.. -----------------------------------------------------------
+
+Pooling Layers
+------------------------
+
+New Pooling Layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: NewPoolingLayer
+```
+
+Finally, test with local documentation:
+```bash
+cd ./docs
+
+make clean
+make html
+# then view generated local documentation by ./html/index.html
+```
+
+## General intro to TensorLayer2
+* TensorLayer2 is built on [TensorFlow2](https://www.tensorflow.org/alpha), so TensorLayer2 is purely eager, no sessions, no globals.
+* TensorLayer2 supports APIs to build static models and dynamic models. Therefore, all `Layers` should be compatible with the two modes.
+```python
+# An example of a static model
+# A static model has inputs and outputs with fixed shape.
+inputs = tl.layers.Input([32, 784])
+dense1 = tl.layers.Dense(n_units=800, act=tf.nn.relu, in_channels=784, name='dense1')(inputs)
+dense2 = tl.layers.Dense(n_units=10, act=tf.nn.relu, in_channels=800, name='dense2')(dense1)
+model = tl.models.Model(inputs=inputs, outputs=dense2)
+
+# An example of a dynamic model
+# A dynamic model has more flexibility. The inputs and outputs may be different in different runs.
+class CustomizeModel(tl.models.Model):
+ def __init__(self):
+ super(CustomizeModel, self).__init__()
+ self.dense1 = tl.layers.Dense(n_units=800, act=tf.nn.relu, in_channels=784, name='dense1')
+ self.dense2 = tl.layers.Dense(n_units=10, act=tf.nn.relu, in_channels=800, name='dense2')
+
+ # a dynamic model allows more flexibility by customising forwarding.
+ def forward(self, x, bar=None):
+ d1 = self.dense1(x)
+ if bar:
+ return d1
+ else:
+ d2 = self.dense2(d1)
+ return d1, d2
+
+model = CustomizeModel()
+```
+* More examples can be found in [examples](examples/) and [tests/layers](tests/layers/). Note that not all of them are completed.
+
+## How to contribute a new `Layer`
+* A `NewLayer` should be a derived from the base class [`Layer`](tensorlayer/layers/core.py).
+* Member methods to be overrided:
+ - `__init__(self, args1, args2, inputs_shape=None, name=None)`: The constructor of the `NewLayer`, which should
+ - Call `super(NewLayer, self).__init__(name)` to construct the base.
+ - Define member variables based on the args1, args2 (or even more).
+ - If the `inputs_shape` is provided, call `self.build(inputs_shape)` and set `self._built=True`. Note that sometimes only `in_channels` should be enough to build the layer like [`Dense`](tensorlayer/layers/dense/base_dense.py).
+ - Logging by `logging.info(...)`.
+ - `__repr__(self)`: Return a printable representation of the `NewLayer`.
+ - `build(self, inputs_shape)`: Build the `NewLayer` by defining weights.
+ - `forward(self, inputs, **kwargs)`: Forward feeding the `NewLayer`. Note that the forward feeding of some `Layers` may be different during training and testing like [`Dropout`](tensorlayer/layers/dropout.py).
+* Unittest:
+ - Unittest should be done before a pull request. Unittest code can be written in [tests/](tests/)
+* Documents:
+ - Please write a description for each class and method in RST format. The description may include the functionality, arguments, references, examples of the `NewLayer`.
+* Examples: [`Dense`](tensorlayer/layers/dense/base_dense.py), [`Dropout`](tensorlayer/layers/dropout.py), [`Conv`](tensorlayer/layers/convolution/simplified_conv.py).
+
+## How to contribute a new `Model`
+* A `NewModel` should be derived from the base class [`Model`](tensorlayer/models/core.py) (if dynamic) or an instance of [`Model`](tensorlayer/models/core.py) (if static).
+* A static `NewModel` should have fixed inputs and outputs. Please check the example [`VGG_Static`](tensorlayer/models/vgg.py)
+* A dynamic `NewModel` has more flexiblility. Please check the example [`VGG16`](tensorlayer/models/vgg16.py)
+
+## How to contribute a new example/tutorial
+* A new example/tutorial should implement a complete workflow of deep learning which includes (but not limited)
+ - `Models` construction based on `Layers`.
+ - Data processing and loading.
+ - Training and testing.
+ - Forward feeding by calling the models.
+ - Loss function.
+ - Back propagation by `tf.GradientTape()`.
+ - Model saving and restoring.
+* Examples: [MNIST](examples/basic_tutorials/tutorial_mnist_mlp_static.py), [CIFAR10](examples/basic_tutorials/tutorial_cifar10_cnn_static.py), [FastText](examples/text_classification/tutorial_imdb_fasttext.py)
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index 4ed90b9..0000000
--- a/LICENSE
+++ /dev/null
@@ -1,208 +0,0 @@
-Apache License
-
-Version 2.0, January 2004
-
-http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION,
-AND DISTRIBUTION
-
- 1. Definitions.
-
-
-
-"License" shall mean the terms and conditions for use, reproduction, and distribution
-as defined by Sections 1 through 9 of this document.
-
-
-
-"Licensor" shall mean the copyright owner or entity authorized by the copyright
-owner that is granting the License.
-
-
-
-"Legal Entity" shall mean the union of the acting entity and all other entities
-that control, are controlled by, or are under common control with that entity.
-For the purposes of this definition, "control" means (i) the power, direct
-or indirect, to cause the direction or management of such entity, whether
-by contract or otherwise, or (ii) ownership of fifty percent (50%) or more
-of the outstanding shares, or (iii) beneficial ownership of such entity.
-
-
-
-"You" (or "Your") shall mean an individual or Legal Entity exercising permissions
-granted by this License.
-
-
-
-"Source" form shall mean the preferred form for making modifications, including
-but not limited to software source code, documentation source, and configuration
-files.
-
-
-
-"Object" form shall mean any form resulting from mechanical transformation
-or translation of a Source form, including but not limited to compiled object
-code, generated documentation, and conversions to other media types.
-
-
-
-"Work" shall mean the work of authorship, whether in Source or Object form,
-made available under the License, as indicated by a copyright notice that
-is included in or attached to the work (an example is provided in the Appendix
-below).
-
-
-
-"Derivative Works" shall mean any work, whether in Source or Object form,
-that is based on (or derived from) the Work and for which the editorial revisions,
-annotations, elaborations, or other modifications represent, as a whole, an
-original work of authorship. For the purposes of this License, Derivative
-Works shall not include works that remain separable from, or merely link (or
-bind by name) to the interfaces of, the Work and Derivative Works thereof.
-
-
-
-"Contribution" shall mean any work of authorship, including the original version
-of the Work and any modifications or additions to that Work or Derivative
-Works thereof, that is intentionally submitted to Licensor for inclusion in
-the Work by the copyright owner or by an individual or Legal Entity authorized
-to submit on behalf of the copyright owner. For the purposes of this definition,
-"submitted" means any form of electronic, verbal, or written communication
-sent to the Licensor or its representatives, including but not limited to
-communication on electronic mailing lists, source code control systems, and
-issue tracking systems that are managed by, or on behalf of, the Licensor
-for the purpose of discussing and improving the Work, but excluding communication
-that is conspicuously marked or otherwise designated in writing by the copyright
-owner as "Not a Contribution."
-
-
-
-"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
-of whom a Contribution has been received by Licensor and subsequently incorporated
-within the Work.
-
-2. Grant of Copyright License. Subject to the terms and conditions of this
-License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
-no-charge, royalty-free, irrevocable copyright license to reproduce, prepare
-Derivative Works of, publicly display, publicly perform, sublicense, and distribute
-the Work and such Derivative Works in Source or Object form.
-
-3. Grant of Patent License. Subject to the terms and conditions of this License,
-each Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
-no-charge, royalty-free, irrevocable (except as stated in this section) patent
-license to make, have made, use, offer to sell, sell, import, and otherwise
-transfer the Work, where such license applies only to those patent claims
-licensable by such Contributor that are necessarily infringed by their Contribution(s)
-alone or by combination of their Contribution(s) with the Work to which such
-Contribution(s) was submitted. If You institute patent litigation against
-any entity (including a cross-claim or counterclaim in a lawsuit) alleging
-that the Work or a Contribution incorporated within the Work constitutes direct
-or contributory patent infringement, then any patent licenses granted to You
-under this License for that Work shall terminate as of the date such litigation
-is filed.
-
-4. Redistribution. You may reproduce and distribute copies of the Work or
-Derivative Works thereof in any medium, with or without modifications, and
-in Source or Object form, provided that You meet the following conditions:
-
-(a) You must give any other recipients of the Work or Derivative Works a copy
-of this License; and
-
-(b) You must cause any modified files to carry prominent notices stating that
-You changed the files; and
-
-(c) You must retain, in the Source form of any Derivative Works that You distribute,
-all copyright, patent, trademark, and attribution notices from the Source
-form of the Work, excluding those notices that do not pertain to any part
-of the Derivative Works; and
-
-(d) If the Work includes a "NOTICE" text file as part of its distribution,
-then any Derivative Works that You distribute must include a readable copy
-of the attribution notices contained within such NOTICE file, excluding those
-notices that do not pertain to any part of the Derivative Works, in at least
-one of the following places: within a NOTICE text file distributed as part
-of the Derivative Works; within the Source form or documentation, if provided
-along with the Derivative Works; or, within a display generated by the Derivative
-Works, if and wherever such third-party notices normally appear. The contents
-of the NOTICE file are for informational purposes only and do not modify the
-License. You may add Your own attribution notices within Derivative Works
-that You distribute, alongside or as an addendum to the NOTICE text from the
-Work, provided that such additional attribution notices cannot be construed
-as modifying the License.
-
-You may add Your own copyright statement to Your modifications and may provide
-additional or different license terms and conditions for use, reproduction,
-or distribution of Your modifications, or for any such Derivative Works as
-a whole, provided Your use, reproduction, and distribution of the Work otherwise
-complies with the conditions stated in this License.
-
-5. Submission of Contributions. Unless You explicitly state otherwise, any
-Contribution intentionally submitted for inclusion in the Work by You to the
-Licensor shall be under the terms and conditions of this License, without
-any additional terms or conditions. Notwithstanding the above, nothing herein
-shall supersede or modify the terms of any separate license agreement you
-may have executed with Licensor regarding such Contributions.
-
-6. Trademarks. This License does not grant permission to use the trade names,
-trademarks, service marks, or product names of the Licensor, except as required
-for reasonable and customary use in describing the origin of the Work and
-reproducing the content of the NOTICE file.
-
-7. Disclaimer of Warranty. Unless required by applicable law or agreed to
-in writing, Licensor provides the Work (and each Contributor provides its
-Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied, including, without limitation, any warranties
-or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR
-A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness
-of using or redistributing the Work and assume any risks associated with Your
-exercise of permissions under this License.
-
-8. Limitation of Liability. In no event and under no legal theory, whether
-in tort (including negligence), contract, or otherwise, unless required by
-applicable law (such as deliberate and grossly negligent acts) or agreed to
-in writing, shall any Contributor be liable to You for damages, including
-any direct, indirect, special, incidental, or consequential damages of any
-character arising as a result of this License or out of the use or inability
-to use the Work (including but not limited to damages for loss of goodwill,
-work stoppage, computer failure or malfunction, or any and all other commercial
-damages or losses), even if such Contributor has been advised of the possibility
-of such damages.
-
-9. Accepting Warranty or Additional Liability. While redistributing the Work
-or Derivative Works thereof, You may choose to offer, and charge a fee for,
-acceptance of support, warranty, indemnity, or other liability obligations
-and/or rights consistent with this License. However, in accepting such obligations,
-You may act only on Your own behalf and on Your sole responsibility, not on
-behalf of any other Contributor, and only if You agree to indemnify, defend,
-and hold each Contributor harmless for any liability incurred by, or claims
-asserted against, such Contributor by reason of your accepting any such warranty
-or additional liability. END OF TERMS AND CONDITIONS
-
-APPENDIX: How to apply the Apache License to your work.
-
-To apply the Apache License to your work, attach the following boilerplate
-notice, with the fields enclosed by brackets "[]" replaced with your own identifying
-information. (Don't include the brackets!) The text should be enclosed in
-the appropriate comment syntax for the file format. We also recommend that
-a file or class name and description of purpose be included on the same "printed
-page" as the copyright notice for easier identification within third-party
-archives.
-
-Copyright [yyyy] [name of copyright owner]
-
-Licensed under the Apache License, Version 2.0 (the "License");
-
-you may not use this file except in compliance with the License.
-
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-
-distributed under the License is distributed on an "AS IS" BASIS,
-
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-
-See the License for the specific language governing permissions and
-
-limitations under the License.
diff --git a/LICENSE.rst b/LICENSE.rst
new file mode 100644
index 0000000..b662f1d
--- /dev/null
+++ b/LICENSE.rst
@@ -0,0 +1,211 @@
+License
+=======
+
+Copyright (c) 2016~2018 The TensorLayer contributors. All rights reserved.
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2016, The TensorLayer Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+Contact
+=======
+Questions? Please contact hao.dong11@imperial.ac.uk
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..4fbfd85
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,35 @@
+default:
+ @echo "Usage:"
+ @echo "\tmake lint # run pylint"
+ @echo "\tmake format # run yapf, autoflake and isort"
+ @echo "\tmake install3 # install tensorlayer in current workspace with pip3"
+
+lint:
+ pylint examples/*.py
+ pylint tensorlayer
+
+test:
+ python3 tests/models/test_model_core.py
+ python3 tests/layers/test_layernode.py
+ python3 tests/files/test_utils_saveload.py
+
+format:
+ autoflake -i examples/*.py
+ autoflake -i tensorlayer/*.py
+ autoflake -i tensorlayer/**/*.py
+
+ isort -rc examples
+ isort -rc tensorlayer
+
+ yapf -i examples/*.py
+ yapf -i tensorlayer/*.py
+ yapf -i tensorlayer/**/*.py
+
+install3:
+ pip3 install -U . --user
+
+
+TAG = tensorlayer-docs:snaphot
+
+doc:
+ docker build --rm -t $(TAG) -f docker/docs/Dockerfile .
diff --git a/README.md b/README.md
index 6034f09..13e3ba0 100644
--- a/README.md
+++ b/README.md
@@ -1,20 +1,181 @@
-#### 从命令行创建一个新的仓库
+
+
+
+
+
+
+
+
+[](https://github.com/tensorflow/tensorflow/releases)
+[](https://tensorlayer.readthedocs.io/)
+[](https://travis-ci.org/tensorlayer/tensorlayer)
+[](http://pepy.tech/project/tensorlayer)
+[](https://hub.docker.com/r/tensorlayer/tensorlayer/)
+[](https://www.codacy.com/app/tensorlayer/tensorlayer)
+
+
+
+
+
+TensorLayer is a novel TensorFlow-based deep learning and reinforcement learning library designed for researchers and engineers. It provides an extensive collection of customizable neural layers to build complex AI models. TensorLayer is awarded the 2017 Best Open Source Software by the [ACM Multimedia Society](https://twitter.com/ImperialDSI/status/923928895325442049).
+TensorLayer can also be found at [iHub](https://code.ihub.org.cn/projects/328) and [Gitee](https://gitee.com/organizations/TensorLayer).
+
+# News
+
+🔥 Reinforcement Learning Model Zoo: [Low-level APIs for Research](https://github.com/tensorlayer/tensorlayer/tree/master/examples/reinforcement_learning) and [High-level APIs for Production](https://github.com/tensorlayer/RLzoo)
+
+🔥 [Sipeed Maxi-EMC](https://github.com/sipeed/Maix-EMC): Run TensorLayer models on the **low-cost AI chip** (e.g., K210) (Alpha Version)
+
+
+
+🔥 [Free GPU and storage resources](https://github.com/fangde/FreeGPU): TensorLayer users can access to free GPU and storage resources donated by SurgicalAI. Thank you SurgicalAI!
+
+# Design Features
+
+TensorLayer is a new deep learning library designed with simplicity, flexibility and high-performance in mind.
+
+- ***Simplicity*** : TensorLayer has a high-level layer/model abstraction which is effortless to learn. You can learn how deep learning can benefit your AI tasks in minutes through the massive [examples](https://github.com/tensorlayer/awesome-tensorlayer).
+- ***Flexibility*** : TensorLayer APIs are transparent and flexible, inspired by the emerging PyTorch library. Compared to the Keras abstraction, TensorLayer makes it much easier to build and train complex AI models.
+- ***Zero-cost Abstraction*** : Though simple to use, TensorLayer does not require you to make any compromise in the performance of TensorFlow (Check the following benchmark section for more details).
+
+TensorLayer stands at a unique spot in the TensorFlow wrappers. Other wrappers like Keras and TFLearn
+hide many powerful features of TensorFlow and provide little support for writing custom AI models. Inspired by PyTorch, TensorLayer APIs are simple, flexible and Pythonic,
+making it easy to learn while being flexible enough to cope with complex AI tasks.
+TensorLayer has a fast-growing community. It has been used by researchers and engineers all over the world, including those from Peking University,
+Imperial College London, UC Berkeley, Carnegie Mellon University, Stanford University, and companies like Google, Microsoft, Alibaba, Tencent, Xiaomi, and Bloomberg.
+
+# Multilingual Documents
+
+TensorLayer has extensive documentation for both beginners and professionals. The documentation is available in
+both English and Chinese.
+
+[](https://tensorlayer.readthedocs.io/)
+[](https://tensorlayercn.readthedocs.io/)
+[](http://www.broadview.com.cn/book/5059/)
+
+If you want to try the experimental features on the the master branch, you can find the latest document
+[here](https://tensorlayer.readthedocs.io/en/latest/).
+
+# Extensive Examples
+
+You can find a large collection of examples that use TensorLayer in [here](examples/) and the following space:
+
+
+
+
+
+
+
+# Citing TensorLayer
+
+If you find TensorLayer useful for your project, please cite the following paper：
+
+```
+@article{tensorlayer2017,
+ author = {Dong, Hao and Supratak, Akara and Mai, Luo and Liu, Fangde and Oehmichen, Axel and Yu, Simiao and Guo, Yike},
+ journal = {ACM Multimedia},
+ title = {{TensorLayer: A Versatile Library for Efficient Deep Learning Development}},
+ url = {http://tensorlayer.org},
+ year = {2017}
+}
+```
\ No newline at end of file
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..5f424cf
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,201 @@
+|TENSORLAYER-LOGO|
+
+
+|Awesome| |Documentation-EN| |Documentation-CN| |Book-CN| |Downloads|
+
+|PyPI| |PyPI-Prerelease| |Commits-Since| |Python| |TensorFlow|
+
+|Travis| |Docker| |RTD-EN| |RTD-CN| |PyUP| |Docker-Pulls| |Code-Quality|
+
+
+|JOIN-SLACK-LOGO|
+
+TensorLayer is a novel TensorFlow-based deep learning and reinforcement
+learning library designed for researchers and engineers. It provides a
+large collection of customizable neural layers / functions that are key
+to build real-world AI applications. TensorLayer is awarded the 2017
+Best Open Source Software by the `ACM Multimedia
+Society `__.
+
+Why another deep learning library: TensorLayer
+==============================================
+
+As deep learning practitioners, we have been looking for a library that
+can address various development purposes. This library is easy to adopt
+by providing diverse examples, tutorials and pre-trained models. Also,
+it allow users to easily fine-tune TensorFlow; while being suitable for
+production deployment. TensorLayer aims to satisfy all these purposes.
+It has three key features:
+
+- **Simplicity** : TensorLayer lifts the low-level dataflow interface
+ of TensorFlow to *high-level* layers / models. It is very easy to
+ learn through the rich `example
+ codes `__
+ contributed by a wide community.
+- **Flexibility** : TensorLayer APIs are transparent: it does not
+ mask TensorFlow from users; but leaving massive hooks that help
+ *low-level tuning* and *deep customization*.
+- **Zero-cost Abstraction** : TensorLayer can achieve the *full
+ power* of TensorFlow. The following table shows the training speeds
+ of classic models using TensorLayer and native TensorFlow on a Titan
+ X Pascal GPU.
+
+ +---------------+-----------------+-----------------+-----------------+
+ | | CIFAR-10 | PTB LSTM | Word2Vec |
+ +===============+=================+=================+=================+
+ | TensorLayer | 2528 images/s | 18063 words/s | 58167 words/s |
+ +---------------+-----------------+-----------------+-----------------+
+ | TensorFlow | 2530 images/s | 18075 words/s | 58181 words/s |
+ +---------------+-----------------+-----------------+-----------------+
+
+TensorLayer stands at a unique spot in the library landscape. Other
+wrapper libraries like Keras and TFLearn also provide high-level
+abstractions. They, however, often hide the underlying engine from
+users, which make them hard to customize and fine-tune. On the contrary,
+TensorLayer APIs are generally flexible and transparent. Users often
+find it easy to start with the examples and tutorials, and then dive
+into TensorFlow seamlessly. In addition, TensorLayer does not create
+library lock-in through native supports for importing components from
+Keras, TFSlim and TFLearn.
+
+TensorLayer has a fast growing usage among top researchers and
+engineers, from universities like Imperial College London, UC Berkeley,
+Carnegie Mellon University, Stanford University, and University of
+Technology of Compiegne (UTC), and companies like Google, Microsoft,
+Alibaba, Tencent, Xiaomi, and Bloomberg.
+
+Install
+=======
+
+TensorLayer has pre-requisites including TensorFlow, numpy, and others. For GPU support, CUDA and cuDNN are required.
+The simplest way to install TensorLayer is to use the Python Package Index (PyPI):
+
+.. code:: bash
+
+ # for last stable version
+ pip install --upgrade tensorlayer
+
+ # for latest release candidate
+ pip install --upgrade --pre tensorlayer
+
+ # if you want to install the additional dependencies, you can also run
+ pip install --upgrade tensorlayer[all] # all additional dependencies
+ pip install --upgrade tensorlayer[extra] # only the `extra` dependencies
+ pip install --upgrade tensorlayer[contrib_loggers] # only the `contrib_loggers` dependencies
+
+Alternatively, you can install the latest or development version by directly pulling from github:
+
+.. code:: bash
+
+ pip install https://github.com/tensorlayer/tensorlayer/archive/master.zip
+ # or
+ # pip install https://github.com/tensorlayer/tensorlayer/archive/.zip
+
+Using Docker - a ready-to-use environment
+-----------------------------------------
+
+The `TensorLayer
+containers `__ are
+built on top of the official `TensorFlow
+containers `__:
+
+Containers with CPU support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+ # for CPU version and Python 2
+ docker pull tensorlayer/tensorlayer:latest
+ docker run -it --rm -p 8888:8888 -p 6006:6006 -e PASSWORD=JUPYTER_NB_PASSWORD tensorlayer/tensorlayer:latest
+
+ # for CPU version and Python 3
+ docker pull tensorlayer/tensorlayer:latest-py3
+ docker run -it --rm -p 8888:8888 -p 6006:6006 -e PASSWORD=JUPYTER_NB_PASSWORD tensorlayer/tensorlayer:latest-py3
+
+Containers with GPU support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+NVIDIA-Docker is required for these containers to work: `Project
+Link `__
+
+.. code:: bash
+
+ # for GPU version and Python 2
+ docker pull tensorlayer/tensorlayer:latest-gpu
+ nvidia-docker run -it --rm -p 8888:88888 -p 6006:6006 -e PASSWORD=JUPYTER_NB_PASSWORD tensorlayer/tensorlayer:latest-gpu
+
+ # for GPU version and Python 3
+ docker pull tensorlayer/tensorlayer:latest-gpu-py3
+ nvidia-docker run -it --rm -p 8888:8888 -p 6006:6006 -e PASSWORD=JUPYTER_NB_PASSWORD tensorlayer/tensorlayer:latest-gpu-py3
+
+Contribute
+==========
+
+Please read the `Contributor
+Guideline `__
+before submitting your PRs.
+
+Cite
+====
+
+If you find this project useful, we would be grateful if you cite the
+TensorLayer paper：
+
+::
+
+ @article{tensorlayer2017,
+ author = {Dong, Hao and Supratak, Akara and Mai, Luo and Liu, Fangde and Oehmichen, Axel and Yu, Simiao and Guo, Yike},
+ journal = {ACM Multimedia},
+ title = {{TensorLayer: A Versatile Library for Efficient Deep Learning Development}},
+ url = {http://tensorlayer.org},
+ year = {2017}
+ }
+
+License
+=======
+
+TensorLayer is released under the Apache 2.0 license.
+
+
+.. |TENSORLAYER-LOGO| image:: https://raw.githubusercontent.com/tensorlayer/tensorlayer/master/img/tl_transparent_logo.png
+ :target: https://tensorlayer.readthedocs.io/
+.. |JOIN-SLACK-LOGO| image:: https://raw.githubusercontent.com/tensorlayer/tensorlayer/master/img/join_slack.png
+ :target: https://join.slack.com/t/tensorlayer/shared_invite/enQtMjUyMjczMzU2Njg4LWI0MWU0MDFkOWY2YjQ4YjVhMzI5M2VlZmE4YTNhNGY1NjZhMzUwMmQ2MTc0YWRjMjQzMjdjMTg2MWQ2ZWJhYzc
+
+.. |Awesome| image:: https://awesome.re/mentioned-badge.svg
+ :target: https://github.com/tensorlayer/awesome-tensorlayer
+.. |Documentation-EN| image:: https://img.shields.io/badge/documentation-english-blue.svg
+ :target: https://tensorlayer.readthedocs.io/
+.. |Documentation-CN| image:: https://img.shields.io/badge/documentation-%E4%B8%AD%E6%96%87-blue.svg
+ :target: https://tensorlayercn.readthedocs.io/
+.. |Book-CN| image:: https://img.shields.io/badge/book-%E4%B8%AD%E6%96%87-blue.svg
+ :target: http://www.broadview.com.cn/book/5059/
+.. |Downloads| image:: http://pepy.tech/badge/tensorlayer
+ :target: http://pepy.tech/project/tensorlayer
+
+
+.. |PyPI| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/github/release/tensorlayer/tensorlayer.svg?label=PyPI%20-%20Release
+ :target: https://pypi.org/project/tensorlayer/
+.. |PyPI-Prerelease| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/github/release/tensorlayer/tensorlayer/all.svg?label=PyPI%20-%20Pre-Release
+ :target: https://pypi.org/project/tensorlayer/
+.. |Commits-Since| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/github/commits-since/tensorlayer/tensorlayer/latest.svg
+ :target: https://github.com/tensorlayer/tensorlayer/compare/1.10.1...master
+.. |Python| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/pypi/pyversions/tensorlayer.svg
+ :target: https://pypi.org/project/tensorlayer/
+.. |TensorFlow| image:: https://img.shields.io/badge/tensorflow-1.6.0+-blue.svg
+ :target: https://github.com/tensorflow/tensorflow/releases
+
+.. |Travis| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/travis/tensorlayer/tensorlayer/master.svg?label=Travis
+ :target: https://travis-ci.org/tensorlayer/tensorlayer
+.. |Docker| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/circleci/project/github/tensorlayer/tensorlayer/master.svg?label=Docker%20Build
+ :target: https://circleci.com/gh/tensorlayer/tensorlayer/tree/master
+.. |RTD-EN| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/readthedocs/tensorlayer/latest.svg?label=ReadTheDocs-EN
+ :target: https://tensorlayer.readthedocs.io/
+.. |RTD-CN| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/readthedocs/tensorlayercn/latest.svg?label=ReadTheDocs-CN
+ :target: https://tensorlayercn.readthedocs.io/
+.. |PyUP| image:: https://pyup.io/repos/github/tensorlayer/tensorlayer/shield.svg
+ :target: https://pyup.io/repos/github/tensorlayer/tensorlayer/
+.. |Docker-Pulls| image:: http://ec2-35-178-47-120.eu-west-2.compute.amazonaws.com/docker/pulls/tensorlayer/tensorlayer.svg
+ :target: https://hub.docker.com/r/tensorlayer/tensorlayer/
+.. |Code-Quality| image:: https://api.codacy.com/project/badge/Grade/d6b118784e25435498e7310745adb848
+ :target: https://www.codacy.com/app/tensorlayer/tensorlayer
diff --git a/docker/DockerLint.bat b/docker/DockerLint.bat
new file mode 100644
index 0000000..3a4b1f8
--- /dev/null
+++ b/docker/DockerLint.bat
@@ -0,0 +1,4 @@
+docker pull hadolint/hadolint:latest
+docker run --rm -i hadolint/hadolint hadolint --ignore DL3007 - < Dockerfile
+
+PAUSE;
\ No newline at end of file
diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
index 0000000..2d3b689
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,35 @@
+# Build args.
+# * Accepted Values:
+# - Python 2 + CPU: "latest" => --build-arg TF_CONTAINER_VERSION="latest"
+# - Python 2 + GPU: "latest-gpu" => --build-arg TF_CONTAINER_VERSION="latest-gpu"
+# - Python 3 + CPU: "latest-py3" => --build-arg TF_CONTAINER_VERSION="latest-py3"
+# - Python 3 + GPU: "latest-gpu-py3" => --build-arg TF_CONTAINER_VERSION="latest-gpu-py3"
+
+ARG TF_CONTAINER_VERSION
+
+FROM tensorflow/tensorflow:${TF_CONTAINER_VERSION}
+
+LABEL version="1.0" maintainer="Jonathan DEKHTIAR "
+
+ARG TL_VERSION
+ARG TF_CONTAINER_VERSION
+
+RUN echo "Container Tag: ${TF_CONTAINER_VERSION}" \
+ && apt-get update \
+ && case $TF_CONTAINER_VERSION in \
+ latest-py3 | latest-gpu-py3) apt-get install -y python3-tk ;; \
+ *) apt-get install -y python-tk ;; \
+ esac \
+ && if [ -z "$TL_VERSION" ]; then \
+ echo "Building a Nightly Release" \
+ && apt-get install -y git \
+ && mkdir /dist/ && cd /dist/ \
+ && git clone https://github.com/tensorlayer/tensorlayer.git \
+ && cd tensorlayer \
+ && pip install --disable-pip-version-check --no-cache-dir --upgrade -e .[all]; \
+ else \
+ echo "Building Tag Release: $TL_VERSION" \
+ && pip install --disable-pip-version-check --no-cache-dir --upgrade tensorlayer[all]=="$TL_VERSION"; \
+ fi \
+ && apt-get autoremove -y \
+ && rm -rf /var/lib/apt/lists/*
diff --git a/docker/docs/Dockerfile b/docker/docs/Dockerfile
new file mode 100644
index 0000000..a20020b
--- /dev/null
+++ b/docker/docs/Dockerfile
@@ -0,0 +1,14 @@
+FROM ubuntu:bionic
+
+ADD docker/docs/sources.list.ustc /etc/apt/sources.list
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt update && \
+ apt install -y python3-pip python3-tk python-qt4 wget && \
+ pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple tensorflow
+ADD . /tensorlayer
+WORKDIR /tensorlayer
+RUN ln -s `which pip3` /usr/bin/pip && \
+ ./scripts/install-horovod-for-doc-test.sh
+RUN pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple .
+RUN pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple -e .[all]
+RUN make -C docs html
diff --git a/docker/docs/sources.list.ustc b/docker/docs/sources.list.ustc
new file mode 100644
index 0000000..e46be90
--- /dev/null
+++ b/docker/docs/sources.list.ustc
@@ -0,0 +1,15 @@
+deb http://mirrors.ustc.edu.cn/ubuntu/ bionic main restricted
+
+deb http://mirrors.ustc.edu.cn/ubuntu/ bionic-updates main restricted
+
+deb http://mirrors.ustc.edu.cn/ubuntu/ bionic universe
+deb http://mirrors.ustc.edu.cn/ubuntu/ bionic-updates universe
+
+deb http://mirrors.ustc.edu.cn/ubuntu/ bionic multiverse
+deb http://mirrors.ustc.edu.cn/ubuntu/ bionic-updates multiverse
+
+deb http://mirrors.ustc.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse
+
+deb http://mirrors.ustc.edu.cn/ubuntu bionic-security main restricted
+deb http://mirrors.ustc.edu.cn/ubuntu bionic-security universe
+deb http://mirrors.ustc.edu.cn/ubuntu bionic-security multiverse
diff --git a/docker/pypi_list.py b/docker/pypi_list.py
new file mode 100644
index 0000000..69a2bad
--- /dev/null
+++ b/docker/pypi_list.py
@@ -0,0 +1,68 @@
+import argparse
+import requests
+import logging
+
+import pip._internal
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser(description='Get the nth version of a given package')
+ parser.add_argument('--package', type=str, required=True, help='The PyPI you want to inspect')
+ parser.add_argument('--nth_last_version', type=int, default=1, help='The nth last package will be retrieved')
+ parser.add_argument('--prerelease', help='Get PreRelease Package Version', action='store_true')
+ parser.add_argument('--debug', help='Print debug information', action='store_true')
+
+ args = parser.parse_args()
+
+ # create logger
+ logger = logging.getLogger("PyPI_CLI")
+
+ formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+
+ ch = logging.StreamHandler()
+ ch.setLevel(logging.DEBUG)
+ ch.setFormatter(formatter)
+ logger.addHandler(ch)
+
+ if args.debug:
+ logger.setLevel(logging.DEBUG)
+
+ logger.debug("Package: %s" % args.package)
+ logger.debug("nth_last_version: %s" % args.nth_last_version)
+ logger.debug("prerelease: %s" % args.prerelease)
+ logger.debug("debug: %s" % args.debug)
+
+ finder = pip._internal.index.PackageFinder(
+ [],
+ ['https://pypi.python.org/simple'],
+ session=requests.Session()
+ )
+ results = finder.find_all_candidates(args.package)
+ tmp_versions = [str(p.version) for p in results]
+
+ logger.debug("%s" % tmp_versions)
+
+ versions = list()
+ for el in tmp_versions:
+ if el not in versions:
+ versions.append(el)
+
+ pos = -1
+ nth_version = 1
+
+ while True:
+ fetched_version = versions[pos]
+
+ logger.debug("Version: %s" % fetched_version)
+
+ if nth_version == args.nth_last_version:
+ if args.prerelease or not ("rc" in fetched_version or "a" in fetched_version or "b" in fetched_version):
+ break
+ else:
+ pos -= 1
+ continue
+
+ pos -= 1
+ nth_version += 1
+
+ print(fetched_version)
diff --git a/docker/version_prefix.py b/docker/version_prefix.py
new file mode 100644
index 0000000..7ee648d
--- /dev/null
+++ b/docker/version_prefix.py
@@ -0,0 +1,37 @@
+import argparse
+import logging
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser(description='Determine the version prefix to apply depending on the version name')
+
+ parser.add_argument(
+ '--version',
+ type=str,
+ required=True,
+ help='The Package Version to be installed in the container'
+ )
+
+ parser.add_argument('--debug', help='Print debug information', action='store_true')
+
+ args = parser.parse_args()
+
+ # create logger
+ logger = logging.getLogger("VERSION_PREFIX_CLI")
+
+ formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+
+ ch = logging.StreamHandler()
+ ch.setLevel(logging.DEBUG)
+ ch.setFormatter(formatter)
+ logger.addHandler(ch)
+
+ if args.debug:
+ logger.setLevel(logging.DEBUG)
+
+ logger.debug("Package Version: %s" % args.version)
+
+ if "rc" in args.version or "a" in args.version or "b" in args.version:
+ print("latest-dev")
+ else:
+ print("latest")
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..6024bcf
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,225 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+BUILDDIR = _build
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+ @echo "Please use \`make ' where is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " singlehtml to make a single large HTML file"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " applehelp to make an Apple Help Book"
+ @echo " devhelp to make HTML files and a Devhelp project"
+ @echo " epub to make an epub"
+ @echo " epub3 to make an epub3"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " latexpdf to make LaTeX files and run them through pdflatex"
+ @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+ @echo " text to make text files"
+ @echo " man to make manual pages"
+ @echo " texinfo to make Texinfo files"
+ @echo " info to make Texinfo files and run them through makeinfo"
+ @echo " gettext to make PO message catalogs"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " xml to make Docutils-native XML files"
+ @echo " pseudoxml to make pseudoxml-XML files for display purposes"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+ @echo " coverage to run coverage check of the documentation (if enabled)"
+ @echo " dummy to check syntax errors of document sources"
+
+.PHONY: clean
+clean:
+ rm -rf $(BUILDDIR)/*
+
+.PHONY: html
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: dirhtml
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+.PHONY: singlehtml
+singlehtml:
+ $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+ @echo
+ @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+.PHONY: pickle
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+.PHONY: json
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+.PHONY: htmlhelp
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+.PHONY: qthelp
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/TLayer.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/TLayer.qhc"
+
+.PHONY: applehelp
+applehelp:
+ $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+ @echo
+ @echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+ @echo "N.B. You won't be able to view it unless you put it in" \
+ "~/Library/Documentation/Help or install it in your application" \
+ "bundle."
+
+.PHONY: devhelp
+devhelp:
+ $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+ @echo
+ @echo "Build finished."
+ @echo "To view the help file:"
+ @echo "# mkdir -p $$HOME/.local/share/devhelp/TLayer"
+ @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/TLayer"
+ @echo "# devhelp"
+
+.PHONY: epub
+epub:
+ $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+ @echo
+ @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+.PHONY: epub3
+epub3:
+ $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3
+ @echo
+ @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3."
+
+.PHONY: latex
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+ @echo "Run \`make' in that directory to run these through (pdf)latex" \
+ "(use \`make latexpdf' here to do that automatically)."
+
+.PHONY: latexpdf
+latexpdf:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through pdflatex..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: latexpdfja
+latexpdfja:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through platex and dvipdfmx..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: text
+text:
+ $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+ @echo
+ @echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+.PHONY: man
+man:
+ $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+ @echo
+ @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+.PHONY: texinfo
+texinfo:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo
+ @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+ @echo "Run \`make' in that directory to run these through makeinfo" \
+ "(use \`make info' here to do that automatically)."
+
+.PHONY: info
+info:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo "Running Texinfo files through makeinfo..."
+ make -C $(BUILDDIR)/texinfo info
+ @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+.PHONY: gettext
+gettext:
+ $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+ @echo
+ @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+.PHONY: changes
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+ @echo
+ @echo "The overview file is in $(BUILDDIR)/changes."
+
+.PHONY: linkcheck
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+ $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+ @echo "Testing of coverage in the sources finished, look at the " \
+ "results in $(BUILDDIR)/coverage/python.txt."
+
+.PHONY: xml
+xml:
+ $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+ @echo
+ @echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+.PHONY: pseudoxml
+pseudoxml:
+ $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+ @echo
+ @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+.PHONY: dummy
+dummy:
+ $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
+ @echo
+ @echo "Build finished. Dummy builder generates no files."
diff --git a/docs/_static/fix_rtd.css b/docs/_static/fix_rtd.css
new file mode 100644
index 0000000..83623a0
--- /dev/null
+++ b/docs/_static/fix_rtd.css
@@ -0,0 +1,8 @@
+/* work around https://github.com/snide/sphinx_rtd_theme/issues/149 */
+.rst-content table.field-list .field-body {
+ padding-top: 8px;
+}
+
+/*.section #basic-2-flip-flop-synchronizer{
+ text-align:justify;
+}*/
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000..89f1601
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,469 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# TensorLayer documentation build configuration file, created by
+# sphinx-quickstart on Tue Aug 2 15:30:55 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os, sys, datetime
+sys.path.insert(0, os.path.abspath("../")) # Important
+sys.path.insert(0, os.path.abspath(os.path.join("..", "tensorlayer"))) # Important
+
+from package_info import __shortversion__
+from package_info import __version__
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+
+# extensions = [
+# 'sphinx.ext.coverage',
+# 'sphinx.ext.githubpages',
+# 'numpydoc',
+# ]
+
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.autosummary',
+ 'sphinx.ext.doctest',
+ 'sphinx.ext.ifconfig',
+ 'sphinx.ext.inheritance_diagram',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.mathjax',
+ 'sphinx.ext.napoleon',
+ 'sphinx.ext.todo',
+ 'sphinx.ext.viewcode',
+]
+
+autodoc_mock_imports = [
+ 'cv2',
+ 'gridfs',
+ 'horovod',
+ 'hyperdash',
+ 'imageio',
+ 'lxml',
+ 'matplotlib',
+ 'nltk',
+ 'numpy',
+ 'PIL',
+ 'progressbar',
+ 'pymongo',
+ 'scipy',
+ 'skimage',
+ 'sklearn',
+ 'tensorflow',
+ 'tqdm',
+ 'h5py',
+
+ # TL C++ Packages
+ 'tensorlayer.third_party.roi_pooling.roi_pooling.roi_pooling_ops',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'TensorLayer'
+copyright = '2016~%s, TensorLayer Contributors' % (str(datetime.datetime.now().year))
+author = 'TensorLayer Contributors'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = __shortversion__
+# The full version, including alpha/beta/rc tags.
+release = __version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#
+# today = ''
+#
+# Else, today_fmt is used as the format for a strftime call.
+#
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+# html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.
+# " v documentation" by default.
+#
+# html_title = 'TensorLayer'
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#
+# html_logo = None
+
+# The name of an image file (relative to this directory) to use as a favicon of
+# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#
+# html_extra_path = []
+
+# If not None, a 'Last updated on:' timestamp is inserted at every page
+# bottom, using the given strftime format.
+# The empty string is equivalent to '%b %d, %Y'.
+#
+# html_last_updated_fmt = None
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+#
+# html_domain_indices = True
+
+# If false, no index is generated.
+#
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
+# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
+#
+# html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# 'ja' uses this config value.
+# 'zh' user can custom change `jieba` dictionary path.
+#
+# html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#
+# html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'TensorLayerdoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'TensorLayer.tex', 'TensorLayer Documentation',
+ 'TensorLayer contributors', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+#
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#
+# latex_appendices = []
+
+# If false, no module index is generated.
+#
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'tensorlayer', 'TensorLayer Documentation',
+ [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'TensorLayer', 'TensorLayer Documentation',
+ author, 'TensorLayer', 'Deep learning and Reinforcement learning library for Researchers and Engineers.',
+ 'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+#
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#
+# texinfo_no_detailmenu = False
+
+
+# -- Options for Epub output ----------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+epub_author = author
+epub_publisher = author
+epub_copyright = copyright
+
+# The basename for the epub file. It defaults to the project name.
+# epub_basename = project
+
+# The HTML theme for the epub output. Since the default themes are not
+# optimized for small screen space, using the same theme for HTML and epub
+# output is usually not wise. This defaults to 'epub', a theme designed to save
+# visual space.
+#
+# epub_theme = 'epub'
+
+# The language of the text. It defaults to the language option
+# or 'en' if the language is not set.
+#
+# epub_language = ''
+
+# The scheme of the identifier. Typical schemes are ISBN or URL.
+# epub_scheme = ''
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A tuple containing the cover image and cover page html template filenames.
+#
+# epub_cover = ()
+
+# A sequence of (type, uri, title) tuples for the guide element of content.opf.
+#
+# epub_guide = ()
+
+# HTML files that should be inserted before the pages created by sphinx.
+# The format is a list of tuples containing the path and title.
+#
+# epub_pre_files = []
+
+# HTML files that should be inserted after the pages created by sphinx.
+# The format is a list of tuples containing the path and title.
+#
+# epub_post_files = []
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+# The depth of the table of contents in toc.ncx.
+#
+# epub_tocdepth = 3
+
+# Allow duplicate toc entries.
+#
+# epub_tocdup = True
+
+# Choose between 'default' and 'includehidden'.
+#
+# epub_tocscope = 'default'
+
+# Fix unsupported image types using the Pillow.
+#
+# epub_fix_images = False
+
+# Scale large images.
+#
+# epub_max_image_width = 0
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#
+# epub_show_urls = 'inline'
+
+# If false, no index is generated.
+#
+# epub_use_index = True
+
+pygments_style = 'sphinx'
+
+html_theme = "sphinx_rtd_theme"
+
+html_theme_path = []
diff --git a/docs/images/affine_transform_comparison.jpg b/docs/images/affine_transform_comparison.jpg
new file mode 100644
index 0000000..f7da0b8
Binary files /dev/null and b/docs/images/affine_transform_comparison.jpg differ
diff --git a/docs/images/affine_transform_why.jpg b/docs/images/affine_transform_why.jpg
new file mode 100644
index 0000000..bd3b905
Binary files /dev/null and b/docs/images/affine_transform_why.jpg differ
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000..977f337
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,100 @@
+Welcome to TensorLayer
+=======================================
+
+
+.. image:: user/my_figs/tl_transparent_logo.png
+ :width: 30 %
+ :align: center
+ :target: https://github.com/tensorlayer/tensorlayer
+
+**Documentation Version:** |release|
+
+**Jun 2019** `Deep Reinforcement Learning Model ZOO Release !! `__.
+
+**Good News:** We won the **Best Open Source Software Award** `@ACM Multimedia (MM) 2017 `_.
+
+`TensorLayer`_ is a Deep Learning (DL) and Reinforcement Learning (RL) library extended from `Google TensorFlow `_. It provides popular DL and RL modules that can be easily customized and assembled for tackling real-world machine learning problems.
+More details can be found `here `_.
+
+.. note::
+ If you got problem to read the docs online, you could download the repository
+ on `GitHub`_, then go to ``/docs/_build/html/index.html`` to read the docs
+ offline. The ``_build`` folder can be generated in ``docs`` using ``make html``.
+
+User Guide
+------------
+
+The TensorLayer user guide explains how to install TensorFlow, CUDA and cuDNN,
+how to build and train neural networks using TensorLayer, and how to contribute
+to the library as a developer.
+
+.. toctree::
+ :maxdepth: 2
+
+ user/installation
+ user/examples
+ user/contributing
+ user/get_involved
+ user/faq
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Getting started
+
+ user/get_start_model
+ user/get_start_advance
+
+API Reference
+-------------
+
+If you are looking for information on a specific function, class or
+method, this part of the documentation is for you.
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Stable Functionalities
+
+ modules/activation
+ modules/array_ops
+ modules/cost
+ modules/prepro
+ modules/files
+ modules/iterate
+ modules/layers
+ modules/models
+ modules/nlp
+ modules/initializers
+ modules/rein
+ modules/utils
+ modules/visualize
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Alpha Version Functionalities
+
+ modules/db
+ modules/optimizers
+ modules/distributed
+
+Command-line Reference
+----------------------
+
+TensorLayer provides a handy command-line tool `tl` to perform some common tasks.
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Command Line Interface
+
+ modules/cli
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
+
+.. _GitHub: https://github.com/tensorlayer/tensorlayer
+.. _TensorLayer: https://github.com/tensorlayer/tensorlayer/
diff --git a/docs/make.bat b/docs/make.bat
new file mode 100644
index 0000000..2daa20d
--- /dev/null
+++ b/docs/make.bat
@@ -0,0 +1,281 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+ set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+ set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+ :help
+ echo.Please use `make ^` where ^ is one of
+ echo. html to make standalone HTML files
+ echo. dirhtml to make HTML files named index.html in directories
+ echo. singlehtml to make a single large HTML file
+ echo. pickle to make pickle files
+ echo. json to make JSON files
+ echo. htmlhelp to make HTML files and a HTML help project
+ echo. qthelp to make HTML files and a qthelp project
+ echo. devhelp to make HTML files and a Devhelp project
+ echo. epub to make an epub
+ echo. epub3 to make an epub3
+ echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+ echo. text to make text files
+ echo. man to make manual pages
+ echo. texinfo to make Texinfo files
+ echo. gettext to make PO message catalogs
+ echo. changes to make an overview over all changed/added/deprecated items
+ echo. xml to make Docutils-native XML files
+ echo. pseudoxml to make pseudoxml-XML files for display purposes
+ echo. linkcheck to check all external links for integrity
+ echo. doctest to run all doctests embedded in the documentation if enabled
+ echo. coverage to run coverage check of the documentation if enabled
+ echo. dummy to check syntax errors of document sources
+ goto end
+)
+
+if "%1" == "clean" (
+ for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+ del /q /s %BUILDDIR%\*
+ goto end
+)
+
+
+REM Check if sphinx-build is available and fallback to Python version if any
+%SPHINXBUILD% 1>NUL 2>NUL
+if errorlevel 9009 goto sphinx_python
+goto sphinx_ok
+
+:sphinx_python
+
+set SPHINXBUILD=python -m sphinx.__init__
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
+ echo.
+ echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+ echo.installed, then set the SPHINXBUILD environment variable to point
+ echo.to the full path of the 'sphinx-build' executable. Alternatively you
+ echo.may add the Sphinx directory to PATH.
+ echo.
+ echo.If you don't have Sphinx installed, grab it from
+ echo.http://sphinx-doc.org/
+ exit /b 1
+)
+
+:sphinx_ok
+
+
+if "%1" == "html" (
+ %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+ goto end
+)
+
+if "%1" == "dirhtml" (
+ %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+ goto end
+)
+
+if "%1" == "singlehtml" (
+ %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+ goto end
+)
+
+if "%1" == "pickle" (
+ %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the pickle files.
+ goto end
+)
+
+if "%1" == "json" (
+ %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the JSON files.
+ goto end
+)
+
+if "%1" == "htmlhelp" (
+ %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+ goto end
+)
+
+if "%1" == "qthelp" (
+ %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+ echo.^> qcollectiongenerator %BUILDDIR%\qthelp\TLayer.qhcp
+ echo.To view the help file:
+ echo.^> assistant -collectionFile %BUILDDIR%\qthelp\TLayer.ghc
+ goto end
+)
+
+if "%1" == "devhelp" (
+ %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished.
+ goto end
+)
+
+if "%1" == "epub" (
+ %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub file is in %BUILDDIR%/epub.
+ goto end
+)
+
+if "%1" == "epub3" (
+ %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub3 file is in %BUILDDIR%/epub3.
+ goto end
+)
+
+if "%1" == "latex" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdf" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf
+ cd %~dp0
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdfja" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf-ja
+ cd %~dp0
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "text" (
+ %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The text files are in %BUILDDIR%/text.
+ goto end
+)
+
+if "%1" == "man" (
+ %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The manual pages are in %BUILDDIR%/man.
+ goto end
+)
+
+if "%1" == "texinfo" (
+ %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+ goto end
+)
+
+if "%1" == "gettext" (
+ %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+ goto end
+)
+
+if "%1" == "changes" (
+ %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.The overview file is in %BUILDDIR%/changes.
+ goto end
+)
+
+if "%1" == "linkcheck" (
+ %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+ goto end
+)
+
+if "%1" == "doctest" (
+ %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+ goto end
+)
+
+if "%1" == "coverage" (
+ %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of coverage in the sources finished, look at the ^
+results in %BUILDDIR%/coverage/python.txt.
+ goto end
+)
+
+if "%1" == "xml" (
+ %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The XML files are in %BUILDDIR%/xml.
+ goto end
+)
+
+if "%1" == "pseudoxml" (
+ %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
+ goto end
+)
+
+if "%1" == "dummy" (
+ %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. Dummy builder generates no files.
+ goto end
+)
+
+:end
diff --git a/docs/modules/activation.rst b/docs/modules/activation.rst
new file mode 100644
index 0000000..3965bd0
--- /dev/null
+++ b/docs/modules/activation.rst
@@ -0,0 +1,73 @@
+API - Activations
+=========================
+
+To make TensorLayer simple, we minimize the number of activation functions as much as
+we can. So we encourage you to use TensorFlow's function. TensorFlow provides
+``tf.nn.relu``, ``tf.nn.relu6``, ``tf.nn.elu``, ``tf.nn.softplus``,
+``tf.nn.softsign`` and so on.
+For parametric activation, please read the layer APIs.
+
+The shortcut of ``tensorlayer.activation`` is ``tensorlayer.act``.
+
+Your activation
+-------------------
+
+Customizes activation function in TensorLayer is very easy.
+The following example implements an activation that multiplies its input by 2.
+For more complex activation, TensorFlow API will be required.
+
+.. code-block:: python
+
+ def double_activation(x):
+ return x * 2
+
+ double_activation = lambda x: x * 2
+
+.. automodule:: tensorlayer.activation
+
+.. autosummary::
+
+ leaky_relu
+ leaky_relu6
+ leaky_twice_relu6
+ ramp
+ swish
+ sign
+ hard_tanh
+ pixel_wise_softmax
+
+Ramp
+------
+.. autofunction:: ramp
+
+Leaky ReLU
+------------
+.. autofunction:: leaky_relu
+
+Leaky ReLU6
+------------
+.. autofunction:: leaky_relu6
+
+Twice Leaky ReLU6
+-----------------
+.. autofunction:: leaky_twice_relu6
+
+Swish
+------------
+.. autofunction:: swish
+
+Sign
+---------------------
+.. autofunction:: sign
+
+Hard Tanh
+---------------------
+.. autofunction:: hard_tanh
+
+Pixel-wise softmax
+--------------------
+.. autofunction:: pixel_wise_softmax
+
+Parametric activation
+------------------------------
+See ``tensorlayer.layers``.
diff --git a/docs/modules/array_ops.rst b/docs/modules/array_ops.rst
new file mode 100644
index 0000000..ee0aff7
--- /dev/null
+++ b/docs/modules/array_ops.rst
@@ -0,0 +1,20 @@
+API - Array Operations
+======================
+
+.. automodule:: tensorlayer.array_ops
+
+.. autosummary::
+
+ alphas
+ alphas_like
+
+Tensorflow Tensor Operations
+----------------------------
+
+tl.alphas
+^^^^^^^^^
+.. autofunction:: alphas
+
+tl.alphas_like
+^^^^^^^^^^^^^^
+.. autofunction:: alphas_like
diff --git a/docs/modules/cli.rst b/docs/modules/cli.rst
new file mode 100644
index 0000000..7fd5af6
--- /dev/null
+++ b/docs/modules/cli.rst
@@ -0,0 +1,6 @@
+CLI - Command Line Interface
+==============================
+
+.. automodule:: tensorlayer.cli
+
+.. automodule:: tensorlayer.cli.train
diff --git a/docs/modules/cost.rst b/docs/modules/cost.rst
new file mode 100644
index 0000000..eba52f4
--- /dev/null
+++ b/docs/modules/cost.rst
@@ -0,0 +1,100 @@
+API - Cost
+==================
+
+To make TensorLayer simple, we minimize the number of cost functions as much as
+we can. So we encourage you to use TensorFlow's function, , see `TensorFlow API `_.
+
+.. note::
+ Please refer to `Getting Started `_ for getting specific weights for weight regularization.
+
+.. automodule:: tensorlayer.cost
+
+.. autosummary::
+
+ cross_entropy
+ sigmoid_cross_entropy
+ binary_cross_entropy
+ mean_squared_error
+ normalized_mean_square_error
+ absolute_difference_error
+ dice_coe
+ dice_hard_coe
+ iou_coe
+ cross_entropy_seq
+ cross_entropy_seq_with_mask
+ cosine_similarity
+ li_regularizer
+ lo_regularizer
+ maxnorm_regularizer
+ maxnorm_o_regularizer
+ maxnorm_i_regularizer
+ huber_loss
+
+
+Softmax cross entropy
+----------------------
+.. autofunction:: cross_entropy
+
+Sigmoid cross entropy
+----------------------
+.. autofunction:: sigmoid_cross_entropy
+
+Binary cross entropy
+-------------------------
+.. autofunction:: binary_cross_entropy
+
+Mean squared error (L2)
+-------------------------
+.. autofunction:: mean_squared_error
+
+Normalized mean square error
+--------------------------------
+.. autofunction:: normalized_mean_square_error
+
+Absolute difference error (L1)
+--------------------------------
+.. autofunction:: absolute_difference_error
+
+Dice coefficient
+-------------------------
+.. autofunction:: dice_coe
+
+Hard Dice coefficient
+-------------------------
+.. autofunction:: dice_hard_coe
+
+IOU coefficient
+-------------------------
+.. autofunction:: iou_coe
+
+Cross entropy for sequence
+-----------------------------
+.. autofunction:: cross_entropy_seq
+
+Cross entropy with mask for sequence
+----------------------------------------
+.. autofunction:: cross_entropy_seq_with_mask
+
+Cosine similarity
+-------------------
+.. autofunction:: cosine_similarity
+
+Regularization functions
+--------------------------
+
+For ``tf.nn.l2_loss``, ``tf.contrib.layers.l1_regularizer``, ``tf.contrib.layers.l2_regularizer`` and
+``tf.contrib.layers.sum_regularizer``, see tensorflow API.
+Maxnorm
+^^^^^^^^^^
+.. autofunction:: maxnorm_regularizer
+
+Special
+^^^^^^^^^^
+.. autofunction:: li_regularizer
+.. autofunction:: lo_regularizer
+.. autofunction:: maxnorm_o_regularizer
+.. autofunction:: maxnorm_i_regularizer
+
+Huber Loss
+^^^^^^^^^^
+.. autofunction:: huber_loss
\ No newline at end of file
diff --git a/docs/modules/db.rst b/docs/modules/db.rst
new file mode 100644
index 0000000..53a89ac
--- /dev/null
+++ b/docs/modules/db.rst
@@ -0,0 +1,260 @@
+API - Database
+=========================
+
+This is the alpha version of database management system.
+If you have any trouble, please ask for help at `tensorlayer@gmail.com `_ .
+
+Why Database
+----------------
+
+TensorLayer is designed for real world production, capable of large scale machine learning applications.
+TensorLayer database is introduced to address the many data management challenges in the large scale machine learning projects, such as:
+
+1. Finding training data from an enterprise data warehouse.
+2. Loading large datasets that are beyond the storage limitation of one computer.
+3. Managing different models with version control, and comparing them(e.g. accuracy).
+4. Automating the process of training, evaluating and deploying machine learning models.
+
+With the TensorLayer system, we introduce this database technology to address the challenges above.
+
+The database management system is designed with the following three principles in mind.
+
+Everything is Data
+^^^^^^^^^^^^^^^^^^
+
+Data warehouses can store and capture the entire machine learning development process. The data can be categorized as:
+
+1. Dataset: This includes all the data used for training, validation and prediction. The labels can be manually specified or generated by model prediction.
+2. Model architecture: The database includes a table that stores different model architectures, enabling users to reuse the many model development works.
+3. Model parameters: This database stores all the model parameters of each epoch in the training step.
+4. Tasks: A project usually include many small tasks. Each task contains the necessary information such as hyper-parameters for training or validation. For a training task, typical information includes training data, the model parameter, the model architecture, how many epochs the training task has. Validation, testing and inference are also supported by the task system.
+5. Loggings: The logs store all the metrics of each machine learning model, such as the time stamp, loss and accuracy of each batch or epoch.
+
+TensorLayer database in principle is a keyword based search engine. Each model, parameter, or training data is assigned many tags.
+The storage system organizes data into two layers: the index layer, and the blob layer. The index layer stores all the tags and references to the blob storage. The index layer is implemented based on NoSQL document database such as MongoDB. The blob layer stores videos, medical images or label masks in large chunk size, which is usually implemented based on a file system. Our database is based on MongoDB. The blob system is based on the GridFS while the indexes are stored as documents.
+
+
+Everything is identified by Query
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Within the database framework, any entity within the data warehouse, such as the data, model or tasks is specified by the database query language.
+As a reference, the query is more space efficient for storage and it can specify multiple objects in a concise way.
+Another advantage of such a design is enabling a highly flexible software system.
+Many system can be implemented by simply rewriting different components, with many new applications can be implemented just by update the query without modification of any application code.
+
+..
+ A pulling based Stream processing pipeline
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ For large training datasets, we provide a stream interface, which can in theory support an unlimitedly large dataset.
+ A stream interface, implemented as a python generators, keeps on generating new data during training.
+ When using the stream interface, the idea of epoch does not apply anymore, instead, we specify the batch size and image a epoch will have a fixed large number of steps.
+
+ Many techniques are introduced behind the stream interface for performance optimization.
+ The stream interface is based on the database cursor technology.
+ For every data query, only the cursors are returned immediately, not the actual query results.
+ The actual data are loaded later when the generators are evaluated.
+
+ The data loading is further optimized in many ways:
+
+ 1. Data are compressed and decompressed,
+ 2. The data are loaded in bulk model to further optimize the IO traffic
+ 3. The data argumentation or random sampling are computed on the fly, only after the data are loaded into the local computer memory.
+ 4. We also introduced simple cache system that stores the recent blob data.
+
+ Based on the stream interface, a continuos machine learning system can be easily implemented.
+ On a distributed system, the model training, validation and deployment can be run by different computing node which are all running continuously.
+ The trainer keeps on optimizing the models with new added data, the evaluation node keeps on evaluating the recent generated models and the deployment system keeps pulling the best models from the our database warehouse for application.
+
+Preparation
+--------------
+
+In principle, the database can be implemented by any document oriented NoSQL database system.
+The existing implementation is based on MongoDB.
+Further implementations on other databases will be released depending on the progress.
+It will be straightforward to port our database system to Google Cloud, AWS and Azure.
+The following tutorials are based on the MongoDB implementation.
+
+Installing and running MongoDB
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The installation instruction of MongoDB can be found at
+`MongoDB Docs `__.
+There are also many MongoDB services from Amazon or GCP, such as Mongo Atlas from MongoDB
+User can also use docker, which is a powerful tool for `deploying software `_ .
+After installing MongoDB, a MongoDB management tool with graphic user interface will be extremely useful.
+Users can also install Studio3T(MongoChef), which is powerful user interface tool for MongoDB and is free for non-commercial use `studio3t `_.
+
+Tutorials
+----------
+
+Connect to the database
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Similar with MongoDB management tools, IP and port number are required for connecting to the database.
+To distinguish the different projects, the database instances have a ``project_name`` argument.
+In the following example, we connect to MongoDB on a local machine with the IP ``localhost``, and port ``27017`` (this is the default port number of MongoDB).
+
+.. code-block:: python
+
+ db = tl.db.TensorHub(ip='localhost', port=27017, dbname='temp',
+ username=None, password='password', project_name='tutorial')
+
+Dataset management
+^^^^^^^^^^^^^^^^^^^^
+
+You can save a dataset into the database and allow all machines to access it.
+Apart from the dataset key, you can also insert a custom argument such as version and description, for better managing the datasets.
+Note that, all saving functions will automatically save a timestamp, allowing you to load staff (data, model, task) using the timestamp.
+
+.. code-block:: python
+
+ db.save_dataset(dataset=[X_train, y_train, X_test, y_test], dataset_name='mnist', description='this is a tutorial')
+
+After saving the dataset, others can access the dataset as followed:
+
+.. code-block:: python
+
+ dataset = db.find_dataset('mnist')
+ dataset = db.find_dataset('mnist', version='1.0')
+
+If you have multiple datasets that use the same dataset key, you can get all of them as followed:
+
+.. code-block:: python
+
+ datasets = db.find_all_datasets('mnist')
+
+
+Model management
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Save model architecture and parameters into database.
+The model architecture is represented by a TL graph, and the parameters are stored as a list of array.
+
+.. code-block:: python
+
+ db.save_model(net, accuracy=0.8, loss=2.3, name='second_model')
+
+After saving the model into database, we can load it as follow:
+
+.. code-block:: python
+
+ net = db.find_model(sess=sess, accuracy=0.8, loss=2.3)
+
+If there are many models, you can use MongoDB's 'sort' method to find the model you want.
+To get the newest or oldest model, you can sort by time:
+
+.. code-block:: python
+
+ ## newest model
+
+ net = db.find_model(sess=sess, sort=[("time", pymongo.DESCENDING)])
+ net = db.find_model(sess=sess, sort=[("time", -1)])
+
+ ## oldest model
+
+ net = db.find_model(sess=sess, sort=[("time", pymongo.ASCENDING)])
+ net = db.find_model(sess=sess, sort=[("time", 1)])
+
+If you save the model along with accuracy, you can get the model with the best accuracy as followed:
+
+.. code-block:: python
+
+ net = db.find_model(sess=sess, sort=[("test_accuracy", -1)])
+
+To delete all models in a project:
+
+.. code-block:: python
+
+ db.delete_model()
+
+If you want to specify which model you want to delete, you need to put arguments inside.
+
+
+Event / Logging management
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Save training log:
+
+.. code-block:: python
+
+ db.save_training_log(accuracy=0.33)
+ db.save_training_log(accuracy=0.44)
+
+Delete logs that match the requirement:
+
+.. code-block:: python
+
+ db.delete_training_log(accuracy=0.33)
+
+Delete all logging of this project:
+
+.. code-block:: python
+
+ db.delete_training_log()
+ db.delete_validation_log()
+ db.delete_testing_log()
+
+Task distribution
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A project usually consists of many tasks such as hyper parameter selection.
+To make it easier, we can distribute these tasks to several GPU servers.
+A task consists of a task script, hyper parameters, desired result and a status.
+
+A task distributor can push both dataset and tasks into a database, allowing task runners on GPU servers to pull and run.
+The following is an example that pushes 3 tasks with different hyper parameters.
+
+.. code-block:: python
+
+ ## save dataset into database, then allow other servers to use it
+ X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 784))
+ db.save_dataset((X_train, y_train, X_val, y_val, X_test, y_test), 'mnist', description='handwriting digit')
+
+ ## push tasks into database, then allow other servers pull tasks to run
+ db.create_task(
+ task_name='mnist', script='task_script.py', hyper_parameters=dict(n_units1=800, n_units2=800),
+ saved_result_keys=['test_accuracy'], description='800-800'
+ )
+
+ db.create_task(
+ task_name='mnist', script='task_script.py', hyper_parameters=dict(n_units1=600, n_units2=600),
+ saved_result_keys=['test_accuracy'], description='600-600'
+ )
+
+ db.create_task(
+ task_name='mnist', script='task_script.py', hyper_parameters=dict(n_units1=400, n_units2=400),
+ saved_result_keys=['test_accuracy'], description='400-400'
+ )
+
+ ## wait for tasks to finish
+ while db.check_unfinished_task(task_name='mnist'):
+ print("waiting runners to finish the tasks")
+ time.sleep(1)
+
+ ## you can get the model and result from database and do some analysis at the end
+
+The task runners on GPU servers can monitor the database, and run the tasks immediately when they are made available.
+In the task script, we can save the final model and results to the database, this allows task distributors to get the desired model and results.
+
+.. code-block:: python
+
+ ## monitors the database and pull tasks to run
+ while True:
+ print("waiting task from distributor")
+ db.run_task(task_name='mnist', sort=[("time", -1)])
+ time.sleep(1)
+
+Example codes
+^^^^^^^^^^^^^^^^
+
+See `here `__.
+
+
+TensorHub API
+---------------------
+
+.. automodule:: tensorlayer.db
+
+.. autoclass:: TensorHub
+ :members:
diff --git a/docs/modules/distributed.rst b/docs/modules/distributed.rst
new file mode 100644
index 0000000..5e30325
--- /dev/null
+++ b/docs/modules/distributed.rst
@@ -0,0 +1,23 @@
+API - Distributed Training
+=============================
+
+(Alpha release - usage might change later)
+
+Helper API to run a distributed training.
+Check these `examples `_.
+
+.. automodule:: tensorlayer.distributed
+
+.. autosummary::
+
+ Trainer
+
+Distributed training
+--------------------
+
+Trainer
+^^^^^^^^^^^
+
+.. autofunction:: Trainer
+
+
diff --git a/docs/modules/files.rst b/docs/modules/files.rst
new file mode 100644
index 0000000..c823b4d
--- /dev/null
+++ b/docs/modules/files.rst
@@ -0,0 +1,295 @@
+API - Files
+===================================
+
+A collections of helper functions to work with dataset.
+Load benchmark dataset, save and restore model, save and load variables.
+
+.. automodule:: tensorlayer.files
+
+.. autosummary::
+
+ load_mnist_dataset
+ load_fashion_mnist_dataset
+ load_cifar10_dataset
+ load_cropped_svhn
+ load_ptb_dataset
+ load_matt_mahoney_text8_dataset
+ load_imdb_dataset
+ load_nietzsche_dataset
+ load_wmt_en_fr_dataset
+ load_flickr25k_dataset
+ load_flickr1M_dataset
+ load_cyclegan_dataset
+ load_celebA_dataset
+ load_voc_dataset
+ load_mpii_pose_dataset
+ download_file_from_google_drive
+
+ save_npz
+ load_npz
+ assign_weights
+ load_and_assign_npz
+ save_npz_dict
+ load_and_assign_npz_dict
+ save_weights_to_hdf5
+ load_hdf5_to_weights_in_order
+ load_hdf5_to_weights
+
+ save_any_to_npy
+ load_npy_to_any
+
+ file_exists
+ folder_exists
+ del_file
+ del_folder
+ read_file
+ load_file_list
+ load_folder_list
+ exists_or_mkdir
+ maybe_download_and_extract
+
+ natural_keys
+
+..
+ save_ckpt
+ load_ckpt
+ save_graph
+ load_graph
+ save_graph_and_params
+ load_graph_and_params
+
+ npz_to_W_pdf
+
+
+Load dataset functions
+------------------------
+
+MNIST
+^^^^^^^
+.. autofunction:: load_mnist_dataset
+
+Fashion-MNIST
+^^^^^^^^^^^^^^^^
+.. autofunction:: load_fashion_mnist_dataset
+
+CIFAR-10
+^^^^^^^^^^^^
+.. autofunction:: load_cifar10_dataset
+
+SVHN
+^^^^^^^
+.. autofunction:: load_cropped_svhn
+
+Penn TreeBank (PTB)
+^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_ptb_dataset
+
+Matt Mahoney's text8
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_matt_mahoney_text8_dataset
+
+IMBD
+^^^^^^^^^^^
+.. autofunction:: load_imdb_dataset
+
+Nietzsche
+^^^^^^^^^^^^^^
+.. autofunction:: load_nietzsche_dataset
+
+English-to-French translation data from the WMT'15 Website
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_wmt_en_fr_dataset
+
+Flickr25k
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_flickr25k_dataset
+
+Flickr1M
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_flickr1M_dataset
+
+CycleGAN
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_cyclegan_dataset
+
+CelebA
+^^^^^^^^^
+.. autofunction:: load_celebA_dataset
+
+VOC 2007/2012
+^^^^^^^^^^^^^^^^
+.. autofunction:: load_voc_dataset
+
+MPII
+^^^^^^^^^^^^^^^^
+.. autofunction:: load_mpii_pose_dataset
+
+Google Drive
+^^^^^^^^^^^^^^^^
+.. autofunction:: download_file_from_google_drive
+
+
+
+
+
+Load and save network
+----------------------
+
+TensorFlow provides ``.ckpt`` file format to save and restore the models, while
+we suggest to use standard python file format ``hdf5`` to save models for the
+sake of cross-platform. Other file formats such as ``.npz`` are also available.
+
+.. code-block:: python
+
+ ## save model as .h5
+ tl.files.save_weights_to_hdf5('model.h5', network.all_weights)
+ # restore model from .h5 (in order)
+ tl.files.load_hdf5_to_weights_in_order('model.h5', network.all_weights)
+ # restore model from .h5 (by name)
+ tl.files.load_hdf5_to_weights('model.h5', network.all_weights)
+
+ ## save model as .npz
+ tl.files.save_npz(network.all_weights , name='model.npz')
+ # restore model from .npz (method 1)
+ load_params = tl.files.load_npz(name='model.npz')
+ tl.files.assign_weights(sess, load_params, network)
+ # restore model from .npz (method 2)
+ tl.files.load_and_assign_npz(sess=sess, name='model.npz', network=network)
+
+ ## you can assign the pre-trained parameters as follow
+ # 1st parameter
+ tl.files.assign_weights(sess, [load_params[0]], network)
+ # the first three parameters
+ tl.files.assign_weights(sess, load_params[:3], network)
+
+Save network into list (npz)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: save_npz
+
+Load network from list (npz)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_npz
+
+Assign a list of parameters to network
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: assign_weights
+
+Load and assign a list of parameters to network
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_and_assign_npz
+
+
+Save network into dict (npz)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: save_npz_dict
+
+Load network from dict (npz)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_and_assign_npz_dict
+
+Save network into OrderedDict (hdf5)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: save_weights_to_hdf5
+
+Load network from hdf5 in order
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_hdf5_to_weights_in_order
+
+Load network from hdf5 by name
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_hdf5_to_weights
+
+..
+ Save network architecture as a graph
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ .. autofunction:: save_graph
+
+ Load network architecture from a graph
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ .. autofunction:: load_graph
+
+ Save network architecture and parameters
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ .. autofunction:: save_graph_and_params
+
+ Load network architecture and parameters
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ .. autofunction:: load_graph_and_params
+
+..
+ Save network into ckpt
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ .. autofunction:: save_ckpt
+
+ Load network from ckpt
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ .. autofunction:: load_ckpt
+
+
+
+
+Load and save variables
+------------------------
+
+Save variables as .npy
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: save_any_to_npy
+
+Load variables from .npy
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_npy_to_any
+
+
+
+
+Folder/File functions
+------------------------
+
+Check file exists
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: file_exists
+
+Check folder exists
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: folder_exists
+
+Delete file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: del_file
+
+Delete folder
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: del_folder
+
+Read file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: read_file
+
+Load file list from folder
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_file_list
+
+Load folder list from folder
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: load_folder_list
+
+Check and Create folder
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: exists_or_mkdir
+
+Download or extract
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: maybe_download_and_extract
+
+
+
+Sort
+-------
+
+List of string with number in human order
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: natural_keys
+
+Visualizing npz file
+----------------------
+.. autofunction:: npz_to_W_pdf
diff --git a/docs/modules/initializers.rst b/docs/modules/initializers.rst
new file mode 100644
index 0000000..6311619
--- /dev/null
+++ b/docs/modules/initializers.rst
@@ -0,0 +1,51 @@
+API - Initializers
+=========================
+
+To make TensorLayer simple, TensorLayer only warps some basic initializers. For more advanced initializer,
+e.g. ``tf.initializers.he_normal``, please refer to TensorFlow provided initializers
+`here `_.
+
+.. automodule:: tensorlayer.initializers
+
+.. autosummary::
+
+ Initializer
+ Zeros
+ Ones
+ Constant
+ RandomUniform
+ RandomNormal
+ TruncatedNormal
+ deconv2d_bilinear_upsampling_initializer
+
+Initializer
+------------
+.. autoclass:: Initializer
+
+Zeros
+------------
+.. autoclass:: Zeros
+
+Ones
+------------
+.. autoclass:: Ones
+
+Constant
+-----------------
+.. autoclass:: Constant
+
+RandomUniform
+--------------
+.. autoclass:: RandomUniform
+
+RandomNormal
+---------------------
+.. autoclass:: RandomNormal
+
+TruncatedNormal
+---------------------
+.. autoclass:: TruncatedNormal
+
+deconv2d_bilinear_upsampling_initializer
+------------------------------------------
+.. autofunction:: deconv2d_bilinear_upsampling_initializer
diff --git a/docs/modules/iterate.rst b/docs/modules/iterate.rst
new file mode 100644
index 0000000..57d2ddc
--- /dev/null
+++ b/docs/modules/iterate.rst
@@ -0,0 +1,36 @@
+API - Iteration
+==========================
+
+Data iteration.
+
+
+.. automodule:: tensorlayer.iterate
+
+.. autosummary::
+
+ minibatches
+ seq_minibatches
+ seq_minibatches2
+ ptb_iterator
+
+
+Non-time series
+--------------------
+
+.. autofunction:: minibatches
+
+
+Time series
+----------------------
+
+Sequence iteration 1
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: seq_minibatches
+
+Sequence iteration 2
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: seq_minibatches2
+
+PTB dataset iteration
+^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: ptb_iterator
diff --git a/docs/modules/layers.rst b/docs/modules/layers.rst
new file mode 100644
index 0000000..f6c86a5
--- /dev/null
+++ b/docs/modules/layers.rst
@@ -0,0 +1,701 @@
+API - Layers
+============
+
+.. automodule:: tensorlayer.layers
+
+.. -----------------------------------------------------------
+.. Layer List
+.. -----------------------------------------------------------
+
+Layer list
+----------
+
+.. autosummary::
+
+ Layer
+
+ Input
+
+ OneHot
+ Word2vecEmbedding
+ Embedding
+ AverageEmbedding
+
+ Dense
+ Dropout
+ GaussianNoise
+ DropconnectDense
+
+ UpSampling2d
+ DownSampling2d
+
+ Conv1d
+ Conv2d
+ Conv3d
+ DeConv2d
+ DeConv3d
+ DepthwiseConv2d
+ SeparableConv1d
+ SeparableConv2d
+ DeformableConv2d
+ GroupConv2d
+
+ PadLayer
+ PoolLayer
+ ZeroPad1d
+ ZeroPad2d
+ ZeroPad3d
+ MaxPool1d
+ MeanPool1d
+ MaxPool2d
+ MeanPool2d
+ MaxPool3d
+ MeanPool3d
+ GlobalMaxPool1d
+ GlobalMeanPool1d
+ GlobalMaxPool2d
+ GlobalMeanPool2d
+ GlobalMaxPool3d
+ GlobalMeanPool3d
+ CornerPool2d
+
+ SubpixelConv1d
+ SubpixelConv2d
+
+ SpatialTransformer2dAffine
+ transformer
+ batch_transformer
+
+ BatchNorm
+ BatchNorm1d
+ BatchNorm2d
+ BatchNorm3d
+ LocalResponseNorm
+ InstanceNorm
+ InstanceNorm1d
+ InstanceNorm2d
+ InstanceNorm3d
+ LayerNorm
+ GroupNorm
+ SwitchNorm
+
+ RNN
+ SimpleRNN
+ GRURNN
+ LSTMRNN
+ BiRNN
+
+ retrieve_seq_length_op
+ retrieve_seq_length_op2
+ retrieve_seq_length_op3
+ target_mask_op
+
+ Flatten
+ Reshape
+ Transpose
+ Shuffle
+
+ Lambda
+
+ Concat
+ Elementwise
+ ElementwiseLambda
+
+ ExpandDims
+ Tile
+
+ Stack
+ UnStack
+
+ Sign
+ Scale
+ BinaryDense
+ BinaryConv2d
+ TernaryDense
+ TernaryConv2d
+ DorefaDense
+ DorefaConv2d
+
+ PRelu
+ PRelu6
+ PTRelu6
+
+ flatten_reshape
+ initialize_rnn_state
+ list_remove_repeat
+
+.. -----------------------------------------------------------
+.. Basic Layers
+.. -----------------------------------------------------------
+
+Base Layer
+-----------
+
+.. autoclass:: Layer
+
+.. -----------------------------------------------------------
+.. Input Layer
+.. -----------------------------------------------------------
+
+Input Layers
+---------------
+
+Input Layer
+^^^^^^^^^^^^^^^^
+.. autofunction:: Input
+
+.. -----------------------------------------------------------
+.. Embedding Layers
+.. -----------------------------------------------------------
+
+
+One-hot Layer
+^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: OneHot
+
+Word2Vec Embedding Layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: Word2vecEmbedding
+
+Embedding Layer
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: Embedding
+
+Average Embedding Layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: AverageEmbedding
+
+.. -----------------------------------------------------------
+.. Activation Layers
+.. -----------------------------------------------------------
+
+
+Activation Layers
+---------------------------
+
+PReLU Layer
+^^^^^^^^^^^^^^^^^
+.. autoclass:: PRelu
+
+
+PReLU6 Layer
+^^^^^^^^^^^^^^^^^^
+.. autoclass:: PRelu6
+
+
+PTReLU6 Layer
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: PTRelu6
+
+
+.. -----------------------------------------------------------
+.. Convolutional Layers
+.. -----------------------------------------------------------
+
+Convolutional Layers
+---------------------
+
+Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Conv1d
+"""""""""""""""""""""
+.. autoclass:: Conv1d
+
+Conv2d
+"""""""""""""""""""""
+.. autoclass:: Conv2d
+
+Conv3d
+"""""""""""""""""""""
+.. autoclass:: Conv3d
+
+Deconvolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+DeConv2d
+"""""""""""""""""""""
+.. autoclass:: DeConv2d
+
+DeConv3d
+"""""""""""""""""""""
+.. autoclass:: DeConv3d
+
+
+Deformable Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+DeformableConv2d
+"""""""""""""""""""""
+.. autoclass:: DeformableConv2d
+
+
+Depthwise Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+DepthwiseConv2d
+"""""""""""""""""""""
+.. autoclass:: DepthwiseConv2d
+
+
+Group Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+GroupConv2d
+"""""""""""""""""""""
+.. autoclass:: GroupConv2d
+
+
+Separable Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+SeparableConv1d
+"""""""""""""""""""""
+.. autoclass:: SeparableConv1d
+
+SeparableConv2d
+"""""""""""""""""""""
+.. autoclass:: SeparableConv2d
+
+
+SubPixel Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+SubpixelConv1d
+"""""""""""""""""""""
+.. autoclass:: SubpixelConv1d
+
+SubpixelConv2d
+"""""""""""""""""""""
+.. autoclass:: SubpixelConv2d
+
+
+.. -----------------------------------------------------------
+.. Dense Layers
+.. -----------------------------------------------------------
+
+Dense Layers
+-------------
+
+Dense Layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: Dense
+
+Drop Connect Dense Layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: DropconnectDense
+
+
+.. -----------------------------------------------------------
+.. Dropout Layer
+.. -----------------------------------------------------------
+
+Dropout Layers
+-------------------
+.. autoclass:: Dropout
+
+.. -----------------------------------------------------------
+.. Extend Layers
+.. -----------------------------------------------------------
+
+Extend Layers
+-------------------
+
+Expand Dims Layer
+^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: ExpandDims
+
+
+Tile layer
+^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: Tile
+
+.. -----------------------------------------------------------
+.. Image Resampling Layers
+.. -----------------------------------------------------------
+
+Image Resampling Layers
+-------------------------
+
+2D UpSampling
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: UpSampling2d
+
+2D DownSampling
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: DownSampling2d
+
+.. -----------------------------------------------------------
+.. Lambda Layer
+.. -----------------------------------------------------------
+
+Lambda Layers
+---------------
+
+Lambda Layer
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: Lambda
+
+ElementWise Lambda Layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: ElementwiseLambda
+
+.. -----------------------------------------------------------
+.. Merge Layer
+.. -----------------------------------------------------------
+
+Merge Layers
+---------------
+
+Concat Layer
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: Concat
+
+ElementWise Layer
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: Elementwise
+
+.. -----------------------------------------------------------
+.. Noise Layers
+.. -----------------------------------------------------------
+
+Noise Layer
+---------------
+.. autoclass:: GaussianNoise
+
+.. -----------------------------------------------------------
+.. Normalization Layers
+.. -----------------------------------------------------------
+
+Normalization Layers
+--------------------
+
+Batch Normalization
+^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: BatchNorm
+
+Batch Normalization 1D
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: BatchNorm1d
+
+Batch Normalization 2D
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: BatchNorm2d
+
+Batch Normalization 3D
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: BatchNorm3d
+
+Local Response Normalization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: LocalResponseNorm
+
+Instance Normalization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: InstanceNorm
+
+Instance Normalization 1D
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: InstanceNorm1d
+
+Instance Normalization 2D
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: InstanceNorm2d
+
+Instance Normalization 3D
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: InstanceNorm3d
+
+Layer Normalization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: LayerNorm
+
+Group Normalization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: GroupNorm
+
+Switch Normalization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: SwitchNorm
+
+.. -----------------------------------------------------------
+.. Padding Layers
+.. -----------------------------------------------------------
+
+Padding Layers
+------------------------
+
+Pad Layer (Expert API)
+^^^^^^^^^^^^^^^^^^^^^^^^^
+Padding layer for any modes.
+
+.. autoclass:: PadLayer
+
+1D Zero padding
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: ZeroPad1d
+
+2D Zero padding
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: ZeroPad2d
+
+3D Zero padding
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: ZeroPad3d
+
+.. -----------------------------------------------------------
+.. Pooling Layers
+.. -----------------------------------------------------------
+
+Pooling Layers
+------------------------
+
+Pool Layer (Expert API)
+^^^^^^^^^^^^^^^^^^^^^^^^^
+Pooling layer for any dimensions and any pooling functions.
+
+.. autoclass:: PoolLayer
+
+1D Max pooling
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: MaxPool1d
+
+1D Mean pooling
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: MeanPool1d
+
+2D Max pooling
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: MaxPool2d
+
+2D Mean pooling
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: MeanPool2d
+
+3D Max pooling
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: MaxPool3d
+
+3D Mean pooling
+^^^^^^^^^^^^^^^^^^^
+.. autoclass:: MeanPool3d
+
+1D Global Max pooling
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: GlobalMaxPool1d
+
+1D Global Mean pooling
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: GlobalMeanPool1d
+
+2D Global Max pooling
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: GlobalMaxPool2d
+
+2D Global Mean pooling
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: GlobalMeanPool2d
+
+3D Global Max pooling
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: GlobalMaxPool3d
+
+3D Global Mean pooling
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: GlobalMeanPool3d
+
+2D Corner pooling
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: CornerPool2d
+
+.. -----------------------------------------------------------
+.. Quantized Layers
+.. -----------------------------------------------------------
+
+Quantized Nets
+------------------
+
+This is an experimental API package for building Quantized Neural Networks. We are using matrix multiplication rather than add-minus and bit-count operation at the moment. Therefore, these APIs would not speed up the inferencing, for production, you can train model via TensorLayer and deploy the model into other customized C/C++ implementation (We probably provide users an extra C/C++ binary net framework that can load model from TensorLayer).
+
+Note that, these experimental APIs can be changed in the future.
+
+
+Sign
+^^^^^^^^^^^^^^
+.. autoclass:: Sign
+
+Scale
+^^^^^^^^^^^^^^
+.. autoclass:: Scale
+
+Binary Dense Layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: BinaryDense
+
+Binary (De)Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+BinaryConv2d
+"""""""""""""""""""""
+.. autoclass:: BinaryConv2d
+
+Ternary Dense Layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+TernaryDense
+"""""""""""""""""""""
+.. autoclass:: TernaryDense
+
+Ternary Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+TernaryConv2d
+"""""""""""""""""""""
+.. autoclass:: TernaryConv2d
+
+DoReFa Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+DorefaConv2d
+"""""""""""""""""""""
+.. autoclass:: DorefaConv2d
+
+DoReFa Convolutions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+DorefaConv2d
+"""""""""""""""""""""
+.. autoclass:: DorefaConv2d
+
+
+.. -----------------------------------------------------------
+.. Recurrent Layers
+.. -----------------------------------------------------------
+
+Recurrent Layers
+---------------------
+
+Common Recurrent layer
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+All recurrent layers can implement any type of RNN cell by feeding different cell function (LSTM, GRU etc).
+
+RNN layer
+""""""""""""""""""""""""""
+.. autoclass:: RNN
+
+RNN layer with Simple RNN Cell
+""""""""""""""""""""""""""""""""""
+.. autoclass:: SimpleRNN
+
+RNN layer with GRU Cell
+""""""""""""""""""""""""""""""""""
+.. autoclass:: GRURNN
+
+RNN layer with LSTM Cell
+""""""""""""""""""""""""""""""""""
+.. autoclass:: LSTMRNN
+
+Bidirectional layer
+"""""""""""""""""""""""""""""""""
+.. autoclass:: BiRNN
+
+Advanced Ops for Dynamic RNN
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+These operations usually be used inside Dynamic RNN layer, they can
+compute the sequence lengths for different situation and get the last RNN outputs by indexing.
+
+Compute Sequence length 1
+""""""""""""""""""""""""""
+.. autofunction:: retrieve_seq_length_op
+
+Compute Sequence length 2
+"""""""""""""""""""""""""""""
+.. autofunction:: retrieve_seq_length_op2
+
+Compute Sequence length 3
+""""""""""""""""""""""""""""
+.. autofunction:: retrieve_seq_length_op3
+
+Compute mask of the target sequence
+"""""""""""""""""""""""""""""""""""""""
+.. autofunction:: target_mask_op
+
+
+
+.. -----------------------------------------------------------
+.. Shape Layers
+.. -----------------------------------------------------------
+
+Shape Layers
+------------
+
+Flatten Layer
+^^^^^^^^^^^^^^^
+.. autoclass:: Flatten
+
+Reshape Layer
+^^^^^^^^^^^^^^^
+.. autoclass:: Reshape
+
+Transpose Layer
+^^^^^^^^^^^^^^^^^
+.. autoclass:: Transpose
+
+Shuffle Layer
+^^^^^^^^^^^^^^^^^
+.. autoclass:: Shuffle
+
+.. -----------------------------------------------------------
+.. Spatial Transformer Layers
+.. -----------------------------------------------------------
+
+Spatial Transformer
+-----------------------
+
+2D Affine Transformation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: SpatialTransformer2dAffine
+
+2D Affine Transformation function
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: transformer
+
+Batch 2D Affine Transformation function
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: batch_transformer
+
+.. -----------------------------------------------------------
+.. Stack Layers
+.. -----------------------------------------------------------
+
+Stack Layer
+-------------
+
+Stack Layer
+^^^^^^^^^^^^^^
+.. autoclass:: Stack
+
+Unstack Layer
+^^^^^^^^^^^^^^^
+.. autoclass:: UnStack
+
+
+.. -----------------------------------------------------------
+.. Helper Functions
+.. -----------------------------------------------------------
+
+Helper Functions
+------------------------
+
+Flatten tensor
+^^^^^^^^^^^^^^^^^
+.. autofunction:: flatten_reshape
+
+Initialize RNN state
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: initialize_rnn_state
+
+Remove repeated items in a list
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: list_remove_repeat
+
diff --git a/docs/modules/models.rst b/docs/modules/models.rst
new file mode 100644
index 0000000..272f1d9
--- /dev/null
+++ b/docs/modules/models.rst
@@ -0,0 +1,59 @@
+API - Models
+================================
+
+TensorLayer provides many pretrained models, you can easily use the whole or a part of the pretrained models via these APIs.
+
+.. automodule:: tensorlayer.models
+
+.. autosummary::
+
+ Model
+
+ VGG16
+ VGG19
+ SqueezeNetV1
+ MobileNetV1
+ ResNet50
+ Seq2seq
+ Seq2seqLuongAttention
+
+
+Base Model
+-----------
+
+.. autoclass:: Model
+
+VGG16
+----------------------
+
+.. autofunction:: VGG16
+
+VGG19
+----------------------
+
+.. autofunction:: VGG19
+
+SqueezeNetV1
+----------------
+.. autofunction:: SqueezeNetV1
+
+MobileNetV1
+----------------
+
+.. autofunction:: MobileNetV1
+
+ResNet50
+----------------
+
+.. autofunction:: ResNet50
+
+Seq2seq
+------------------------
+
+.. autoclass:: Seq2seq
+
+
+Seq2seq Luong Attention
+------------------------
+
+.. autoclass:: Seq2seqLuongAttention
diff --git a/docs/modules/nlp.rst b/docs/modules/nlp.rst
new file mode 100644
index 0000000..16486a6
--- /dev/null
+++ b/docs/modules/nlp.rst
@@ -0,0 +1,148 @@
+API - Natural Language Processing
+==================================
+
+Natural Language Processing and Word Representation.
+
+.. automodule:: tensorlayer.nlp
+
+.. autosummary::
+
+ generate_skip_gram_batch
+
+ sample
+ sample_top
+
+ SimpleVocabulary
+ Vocabulary
+ process_sentence
+ create_vocab
+
+ simple_read_words
+ read_words
+ read_analogies_file
+ build_vocab
+ build_reverse_dictionary
+ build_words_dataset
+ save_vocab
+
+ words_to_word_ids
+ word_ids_to_words
+
+ basic_tokenizer
+ create_vocabulary
+ initialize_vocabulary
+ sentence_to_token_ids
+ data_to_token_ids
+
+ moses_multi_bleu
+
+
+Iteration function for training embedding matrix
+-------------------------------------------------
+.. autofunction:: generate_skip_gram_batch
+
+
+Sampling functions
+-------------------
+
+Simple sampling
+^^^^^^^^^^^^^^^^^^^
+.. autofunction:: sample
+
+Sampling from top k
+^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: sample_top
+
+Vector representations of words
+-------------------------------
+
+Simple vocabulary class
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: SimpleVocabulary
+
+Vocabulary class
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: Vocabulary
+
+Process sentence
+^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: process_sentence
+
+Create vocabulary
+^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: create_vocab
+
+Read words from file
+----------------------
+
+Simple read file
+^^^^^^^^^^^^^^^^^^
+.. autofunction:: simple_read_words
+
+Read file
+^^^^^^^^^^^^^^^^^^
+.. autofunction:: read_words
+
+
+Read analogy question file
+-----------------------------
+.. autofunction:: read_analogies_file
+
+Build vocabulary, word dictionary and word tokenization
+--------------------------------------------------------
+
+Build dictionary from word to id
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: build_vocab
+
+Build dictionary from id to word
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: build_reverse_dictionary
+
+Build dictionaries for id to word etc
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: build_words_dataset
+
+Save vocabulary
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: save_vocab
+
+
+Convert words to IDs and IDs to words
+--------------------------------------------------------
+These functions can be done by ``Vocabulary`` class.
+
+List of Words to IDs
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: words_to_word_ids
+
+List of IDs to Words
+^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: word_ids_to_words
+
+
+
+Functions for translation
+---------------------------
+
+Word Tokenization
+^^^^^^^^^^^^^^^^^^^
+.. autofunction:: basic_tokenizer
+
+Create or read vocabulary
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: create_vocabulary
+.. autofunction:: initialize_vocabulary
+
+Convert words to IDs and IDs to words
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: sentence_to_token_ids
+.. autofunction:: data_to_token_ids
+
+
+Metrics
+---------------------------
+
+BLEU
+^^^^^^^^^^^^^^^^^^^
+.. autofunction:: moses_multi_bleu
diff --git a/docs/modules/optimizers.rst b/docs/modules/optimizers.rst
new file mode 100644
index 0000000..0ababc8
--- /dev/null
+++ b/docs/modules/optimizers.rst
@@ -0,0 +1,19 @@
+API - Optimizers
+================
+
+.. automodule:: tensorlayer.optimizers
+
+TensorLayer provides simple API and tools to ease research, development and reduce the time to production.
+Therefore, we provide the latest state of the art optimizers that work with Tensorflow.
+
+Optimizers List
+---------------
+
+.. autosummary::
+
+ AMSGrad
+
+AMSGrad Optimizer
+-----------------
+.. autoclass:: AMSGrad
+ :members:
\ No newline at end of file
diff --git a/docs/modules/prepro.rst b/docs/modules/prepro.rst
new file mode 100644
index 0000000..ec65da0
--- /dev/null
+++ b/docs/modules/prepro.rst
@@ -0,0 +1,641 @@
+API - Data Pre-Processing
+=========================
+
+.. automodule:: tensorlayer.prepro
+
+.. autosummary::
+
+ affine_rotation_matrix
+ affine_horizontal_flip_matrix
+ affine_vertical_flip_matrix
+ affine_shift_matrix
+ affine_shear_matrix
+ affine_zoom_matrix
+ affine_respective_zoom_matrix
+
+ transform_matrix_offset_center
+ affine_transform
+ affine_transform_cv2
+ affine_transform_keypoints
+ projective_transform_by_points
+
+ rotation
+ rotation_multi
+ crop
+ crop_multi
+ flip_axis
+ flip_axis_multi
+ shift
+ shift_multi
+
+ shear
+ shear_multi
+ shear2
+ shear_multi2
+ swirl
+ swirl_multi
+ elastic_transform
+ elastic_transform_multi
+
+ zoom
+ respective_zoom
+ zoom_multi
+
+ brightness
+ brightness_multi
+
+ illumination
+
+ rgb_to_hsv
+ hsv_to_rgb
+ adjust_hue
+
+ imresize
+
+ pixel_value_scale
+
+ samplewise_norm
+ featurewise_norm
+
+ channel_shift
+ channel_shift_multi
+
+ drop
+
+ array_to_img
+
+ find_contours
+ pt2map
+ binary_dilation
+ dilation
+ binary_erosion
+ erosion
+
+
+ obj_box_coord_rescale
+ obj_box_coords_rescale
+ obj_box_coord_scale_to_pixelunit
+ obj_box_coord_centroid_to_upleft_butright
+ obj_box_coord_upleft_butright_to_centroid
+ obj_box_coord_centroid_to_upleft
+ obj_box_coord_upleft_to_centroid
+
+ parse_darknet_ann_str_to_list
+ parse_darknet_ann_list_to_cls_box
+
+ obj_box_horizontal_flip
+ obj_box_imresize
+ obj_box_crop
+ obj_box_shift
+ obj_box_zoom
+
+ keypoint_random_crop
+ keypoint_resize_random_crop
+ keypoint_random_rotate
+ keypoint_random_flip
+ keypoint_random_resize
+ keypoint_random_resize_shortestedge
+
+ pad_sequences
+ remove_pad_sequences
+ process_sequences
+ sequences_add_start_id
+ sequences_add_end_id
+ sequences_add_end_id_after_pad
+ sequences_get_mask
+
+
+..
+ Threading
+ ------------
+ .. autofunction:: threading_data
+
+
+Affine Transform
+----------------
+
+
+Python can be FAST
+^^^^^^^^^^^^^^^^^^
+
+Image augmentation is a critical step in deep learning.
+Though TensorFlow has provided ``tf.image``,
+image augmentation often remains as a key bottleneck.
+``tf.image`` has three limitations:
+
+- Real-world visual tasks such as object detection, segmentation, and pose estimation
+ must cope with image meta-data (e.g., coordinates).
+ These data are beyond ``tf.image``
+ which processes images as tensors.
+
+- ``tf.image`` operators
+ breaks the pure Python programing experience (i.e., users have to
+ use ``tf.py_func`` in order to call image functions written in Python); however,
+ frequent uses of ``tf.py_func`` slow down TensorFlow,
+ making users hard to balance flexibility and performance.
+
+- ``tf.image`` API is inflexible. Image operations are
+ performed in an order. They are hard to jointly optimize. More importantly,
+ sequential image operations can significantly
+ reduces the quality of images, thus affecting training accuracy.
+
+
+TensorLayer addresses these limitations by providing a
+high-performance image augmentation API in Python.
+This API bases on affine transformation and ``cv2.wrapAffine``.
+It allows you to combine multiple image processing functions into
+a single matrix operation. This combined operation
+is executed by the fast ``cv2`` library, offering 78x performance improvement (observed in
+`openpose-plus `_ for example).
+The following example illustrates the rationale
+behind this tremendous speed up.
+
+
+Example
+^^^^^^^
+
+The source code of complete examples can be found \
+`here `__.
+The following is a typical Python program that applies rotation, shifting, flipping, zooming and shearing to an image,
+
+.. code-block:: python
+
+ image = tl.vis.read_image('tiger.jpeg')
+
+ xx = tl.prepro.rotation(image, rg=-20, is_random=False)
+ xx = tl.prepro.flip_axis(xx, axis=1, is_random=False)
+ xx = tl.prepro.shear2(xx, shear=(0., -0.2), is_random=False)
+ xx = tl.prepro.zoom(xx, zoom_range=0.8)
+ xx = tl.prepro.shift(xx, wrg=-0.1, hrg=0, is_random=False)
+
+ tl.vis.save_image(xx, '_result_slow.png')
+
+
+However, by leveraging affine transformation, image operations can be combined into one:
+
+.. code-block:: python
+
+ # 1. Create required affine transformation matrices
+ M_rotate = tl.prepro.affine_rotation_matrix(angle=20)
+ M_flip = tl.prepro.affine_horizontal_flip_matrix(prob=1)
+ M_shift = tl.prepro.affine_shift_matrix(wrg=0.1, hrg=0, h=h, w=w)
+ M_shear = tl.prepro.affine_shear_matrix(x_shear=0.2, y_shear=0)
+ M_zoom = tl.prepro.affine_zoom_matrix(zoom_range=0.8)
+
+ # 2. Combine matrices
+ # NOTE: operations are applied in a reversed order (i.e., rotation is performed first)
+ M_combined = M_shift.dot(M_zoom).dot(M_shear).dot(M_flip).dot(M_rotate)
+
+ # 3. Convert the matrix from Cartesian coordinates (the origin in the middle of image)
+ # to image coordinates (the origin on the top-left of image)
+ transform_matrix = tl.prepro.transform_matrix_offset_center(M_combined, x=w, y=h)
+
+ # 4. Transform the image using a single operation
+ result = tl.prepro.affine_transform_cv2(image, transform_matrix) # 76 times faster
+
+ tl.vis.save_image(result, '_result_fast.png')
+
+
+The following figure illustrates the rational behind combined affine transformation.
+
+.. image:: ../images/affine_transform_why.jpg
+ :width: 100 %
+ :align: center
+
+
+Using combined affine transformation has two key benefits. First, it allows \
+you to leverage a pure Python API to achieve orders of magnitudes of speed up in image augmentation,
+and thus prevent data pre-processing from becoming a bottleneck in training. \
+Second, performing sequential image transformation requires multiple image interpolations. \
+This produces low-quality input images. In contrast, a combined transformation performs the \
+interpolation only once, and thus
+preserve the content in an image. The following figure illustrates these two benefits:
+
+.. image:: ../images/affine_transform_comparison.jpg
+ :width: 100 %
+ :align: center
+
+The major reason for combined affine transformation being fast is because it has lower computational complexity.
+Assume we have ``k`` affine transformations ``T1, ..., Tk``, where ``Ti`` can be represented by 3x3 matrixes.
+The sequential transformation can be represented as ``y = Tk (... T1(x))``,
+and the time complexity is ``O(k N)`` where ``N`` is the cost of applying one transformation to image ``x``.
+``N`` is linear to the size of ``x``.
+For the combined transformation ``y = (Tk ... T1) (x)``
+the time complexity is ``O(27(k - 1) + N) = max{O(27k), O(N)} = O(N)`` (assuming 27k << N) where 27 = 3^3 is the cost for combining two transformations.
+
+
+Get rotation matrix
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_rotation_matrix
+
+Get horizontal flipping matrix
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_horizontal_flip_matrix
+
+Get vertical flipping matrix
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_vertical_flip_matrix
+
+Get shifting matrix
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_shift_matrix
+
+Get shearing matrix
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_shear_matrix
+
+Get zooming matrix
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_zoom_matrix
+
+Get respective zooming matrix
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_respective_zoom_matrix
+
+Cartesian to image coordinates
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: transform_matrix_offset_center
+
+..
+ Apply image transform
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ .. autofunction:: affine_transform
+
+Apply image transform
+^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_transform_cv2
+
+Apply keypoint transform
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: affine_transform_keypoints
+
+
+Images
+-----------
+
+Projective transform by points
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: projective_transform_by_points
+
+Rotation
+^^^^^^^^^
+.. autofunction:: rotation
+.. autofunction:: rotation_multi
+
+Crop
+^^^^^^^^^
+.. autofunction:: crop
+.. autofunction:: crop_multi
+
+Flip
+^^^^^^^^^
+.. autofunction:: flip_axis
+.. autofunction:: flip_axis_multi
+
+Shift
+^^^^^^^^^
+.. autofunction:: shift
+.. autofunction:: shift_multi
+
+Shear
+^^^^^^^^^
+.. autofunction:: shear
+.. autofunction:: shear_multi
+
+Shear V2
+^^^^^^^^^^^
+.. autofunction:: shear2
+.. autofunction:: shear_multi2
+
+Swirl
+^^^^^^^^^
+.. autofunction:: swirl
+.. autofunction:: swirl_multi
+
+Elastic transform
+^^^^^^^^^^^^^^^^^^
+.. autofunction:: elastic_transform
+.. autofunction:: elastic_transform_multi
+
+Zoom
+^^^^^^^^^
+.. autofunction:: zoom
+.. autofunction:: zoom_multi
+
+Respective Zoom
+^^^^^^^^^^^^^^^^^
+.. autofunction:: respective_zoom
+
+Brightness
+^^^^^^^^^^^^
+.. autofunction:: brightness
+.. autofunction:: brightness_multi
+
+Brightness, contrast and saturation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: illumination
+
+RGB to HSV
+^^^^^^^^^^^^^^
+.. autofunction:: rgb_to_hsv
+
+HSV to RGB
+^^^^^^^^^^^^^^
+.. autofunction:: hsv_to_rgb
+
+Adjust Hue
+^^^^^^^^^^^^^^
+.. autofunction:: adjust_hue
+
+Resize
+^^^^^^^^^^^^
+.. autofunction:: imresize
+
+Pixel value scale
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: pixel_value_scale
+
+Normalization
+^^^^^^^^^^^^^^^
+.. autofunction:: samplewise_norm
+.. autofunction:: featurewise_norm
+
+Channel shift
+^^^^^^^^^^^^^^
+.. autofunction:: channel_shift
+.. autofunction:: channel_shift_multi
+
+Noise
+^^^^^^^^^^^^^^
+.. autofunction:: drop
+
+Numpy and PIL
+^^^^^^^^^^^^^^
+.. autofunction:: array_to_img
+
+Find contours
+^^^^^^^^^^^^^^
+.. autofunction:: find_contours
+
+Points to Image
+^^^^^^^^^^^^^^^^^
+.. autofunction:: pt2map
+
+Binary dilation
+^^^^^^^^^^^^^^^^^
+.. autofunction:: binary_dilation
+
+Greyscale dilation
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: dilation
+
+Binary erosion
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: binary_erosion
+
+Greyscale erosion
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: erosion
+
+
+
+Object detection
+-------------------
+
+Tutorial for Image Aug
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Hi, here is an example for image augmentation on VOC dataset.
+
+.. code-block:: python
+
+ import tensorlayer as tl
+
+ ## download VOC 2012 dataset
+ imgs_file_list, _, _, _, classes, _, _,\
+ _, objs_info_list, _ = tl.files.load_voc_dataset(dataset="2012")
+
+ ## parse annotation and convert it into list format
+ ann_list = []
+ for info in objs_info_list:
+ ann = tl.prepro.parse_darknet_ann_str_to_list(info)
+ c, b = tl.prepro.parse_darknet_ann_list_to_cls_box(ann)
+ ann_list.append([c, b])
+
+ # read and save one image
+ idx = 2 # you can select your own image
+ image = tl.vis.read_image(imgs_file_list[idx])
+ tl.vis.draw_boxes_and_labels_to_image(image, ann_list[idx][0],
+ ann_list[idx][1], [], classes, True, save_name='_im_original.png')
+
+ # left right flip
+ im_flip, coords = tl.prepro.obj_box_horizontal_flip(image,
+ ann_list[idx][1], is_rescale=True, is_center=True, is_random=False)
+ tl.vis.draw_boxes_and_labels_to_image(im_flip, ann_list[idx][0],
+ coords, [], classes, True, save_name='_im_flip.png')
+
+ # resize
+ im_resize, coords = tl.prepro.obj_box_imresize(image,
+ coords=ann_list[idx][1], size=[300, 200], is_rescale=True)
+ tl.vis.draw_boxes_and_labels_to_image(im_resize, ann_list[idx][0],
+ coords, [], classes, True, save_name='_im_resize.png')
+
+ # crop
+ im_crop, clas, coords = tl.prepro.obj_box_crop(image, ann_list[idx][0],
+ ann_list[idx][1], wrg=200, hrg=200,
+ is_rescale=True, is_center=True, is_random=False)
+ tl.vis.draw_boxes_and_labels_to_image(im_crop, clas, coords, [],
+ classes, True, save_name='_im_crop.png')
+
+ # shift
+ im_shfit, clas, coords = tl.prepro.obj_box_shift(image, ann_list[idx][0],
+ ann_list[idx][1], wrg=0.1, hrg=0.1,
+ is_rescale=True, is_center=True, is_random=False)
+ tl.vis.draw_boxes_and_labels_to_image(im_shfit, clas, coords, [],
+ classes, True, save_name='_im_shift.png')
+
+ # zoom
+ im_zoom, clas, coords = tl.prepro.obj_box_zoom(image, ann_list[idx][0],
+ ann_list[idx][1], zoom_range=(1.3, 0.7),
+ is_rescale=True, is_center=True, is_random=False)
+ tl.vis.draw_boxes_and_labels_to_image(im_zoom, clas, coords, [],
+ classes, True, save_name='_im_zoom.png')
+
+
+In practice, you may want to use threading method to process a batch of images as follows.
+
+.. code-block:: python
+
+ import tensorlayer as tl
+ import random
+
+ batch_size = 64
+ im_size = [416, 416]
+ n_data = len(imgs_file_list)
+ jitter = 0.2
+ def _data_pre_aug_fn(data):
+ im, ann = data
+ clas, coords = ann
+ ## change image brightness, contrast and saturation randomly
+ im = tl.prepro.illumination(im, gamma=(0.5, 1.5),
+ contrast=(0.5, 1.5), saturation=(0.5, 1.5), is_random=True)
+ ## flip randomly
+ im, coords = tl.prepro.obj_box_horizontal_flip(im, coords,
+ is_rescale=True, is_center=True, is_random=True)
+ ## randomly resize and crop image, it can have same effect as random zoom
+ tmp0 = random.randint(1, int(im_size[0]*jitter))
+ tmp1 = random.randint(1, int(im_size[1]*jitter))
+ im, coords = tl.prepro.obj_box_imresize(im, coords,
+ [im_size[0]+tmp0, im_size[1]+tmp1], is_rescale=True,
+ interp='bicubic')
+ im, clas, coords = tl.prepro.obj_box_crop(im, clas, coords,
+ wrg=im_size[1], hrg=im_size[0], is_rescale=True,
+ is_center=True, is_random=True)
+ ## rescale value from [0, 255] to [-1, 1] (optional)
+ im = im / 127.5 - 1
+ return im, [clas, coords]
+
+ # randomly read a batch of image and the corresponding annotations
+ idexs = tl.utils.get_random_int(min=0, max=n_data-1, number=batch_size)
+ b_im_path = [imgs_file_list[i] for i in idexs]
+ b_images = tl.prepro.threading_data(b_im_path, fn=tl.vis.read_image)
+ b_ann = [ann_list[i] for i in idexs]
+
+ # threading process
+ data = tl.prepro.threading_data([_ for _ in zip(b_images, b_ann)],
+ _data_pre_aug_fn)
+ b_images2 = [d[0] for d in data]
+ b_ann = [d[1] for d in data]
+
+ # save all images
+ for i in range(len(b_images)):
+ tl.vis.draw_boxes_and_labels_to_image(b_images[i],
+ ann_list[idexs[i]][0], ann_list[idexs[i]][1], [],
+ classes, True, save_name='_bbox_vis_%d_original.png' % i)
+ tl.vis.draw_boxes_and_labels_to_image((b_images2[i]+1)*127.5,
+ b_ann[i][0], b_ann[i][1], [], classes, True,
+ save_name='_bbox_vis_%d.png' % i)
+
+Image Aug with TF Dataset API
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+- Example code for VOC `here `__.
+
+Coordinate pixel unit to percentage
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_coord_rescale
+
+Coordinates pixel unit to percentage
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_coords_rescale
+
+Coordinate percentage to pixel unit
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_coord_scale_to_pixelunit
+
+Coordinate [x_center, x_center, w, h] to up-left button-right
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_coord_centroid_to_upleft_butright
+
+Coordinate up-left button-right to [x_center, x_center, w, h]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_coord_upleft_butright_to_centroid
+
+Coordinate [x_center, x_center, w, h] to up-left-width-high
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_coord_centroid_to_upleft
+
+Coordinate up-left-width-high to [x_center, x_center, w, h]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_coord_upleft_to_centroid
+
+Darknet format string to list
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: parse_darknet_ann_str_to_list
+
+Darknet format split class and coordinate
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: parse_darknet_ann_list_to_cls_box
+
+Image Aug - Flip
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_horizontal_flip
+
+Image Aug - Resize
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_imresize
+
+Image Aug - Crop
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_crop
+
+Image Aug - Shift
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_shift
+
+Image Aug - Zoom
+^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: obj_box_zoom
+
+Keypoints
+------------
+
+Image Aug - Crop
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: keypoint_random_crop
+
+Image Aug - Resize then Crop
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: keypoint_resize_random_crop
+
+Image Aug - Rotate
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: keypoint_random_rotate
+
+Image Aug - Flip
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: keypoint_random_flip
+
+Image Aug - Resize
+^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: keypoint_random_resize
+
+Image Aug - Resize Shortest Edge
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: keypoint_random_resize_shortestedge
+
+
+Sequence
+---------
+
+More related functions can be found in ``tensorlayer.nlp``.
+
+Padding
+^^^^^^^^^
+.. autofunction:: pad_sequences
+
+Remove Padding
+^^^^^^^^^^^^^^^^^
+.. autofunction:: remove_pad_sequences
+
+
+Process
+^^^^^^^^^
+.. autofunction:: process_sequences
+
+Add Start ID
+^^^^^^^^^^^^^^^
+.. autofunction:: sequences_add_start_id
+
+
+Add End ID
+^^^^^^^^^^^^^^^
+.. autofunction:: sequences_add_end_id
+
+Add End ID after pad
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: sequences_add_end_id_after_pad
+
+Get Mask
+^^^^^^^^^
+.. autofunction:: sequences_get_mask
diff --git a/docs/modules/rein.rst b/docs/modules/rein.rst
new file mode 100644
index 0000000..9ee16a6
--- /dev/null
+++ b/docs/modules/rein.rst
@@ -0,0 +1,33 @@
+API - Reinforcement Learning
+==============================
+
+Reinforcement Learning.
+
+.. automodule:: tensorlayer.rein
+
+.. autosummary::
+
+ discount_episode_rewards
+ cross_entropy_reward_loss
+ log_weight
+ choice_action_by_probs
+
+
+Reward functions
+---------------------
+.. autofunction:: discount_episode_rewards
+
+Cost functions
+---------------------
+
+Weighted Cross Entropy
+^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: cross_entropy_reward_loss
+
+Log weight
+^^^^^^^^^^^^^^
+.. autofunction:: log_weight
+
+Sampling functions
+---------------------
+.. autofunction:: choice_action_by_probs
diff --git a/docs/modules/utils.rst b/docs/modules/utils.rst
new file mode 100644
index 0000000..dfc5f6b
--- /dev/null
+++ b/docs/modules/utils.rst
@@ -0,0 +1,73 @@
+API - Utility
+========================
+
+.. automodule:: tensorlayer.utils
+
+.. autosummary::
+
+ fit
+ test
+ predict
+ evaluation
+ class_balancing_oversample
+ get_random_int
+ dict_to_one
+ list_string_to_dict
+ flatten_list
+ exit_tensorflow
+ open_tensorboard
+ set_gpu_fraction
+
+Training, testing and predicting
+----------------------------------
+
+Training
+^^^^^^^^^^^
+.. autofunction:: fit
+
+Evaluation
+^^^^^^^^^^^^^
+.. autofunction:: test
+
+Prediction
+^^^^^^^^^^^^
+.. autofunction:: predict
+
+Evaluation functions
+---------------------
+.. autofunction:: evaluation
+
+Class balancing functions
+----------------------------
+.. autofunction:: class_balancing_oversample
+
+Random functions
+----------------------------
+.. autofunction:: get_random_int
+
+Dictionary and list
+--------------------
+
+Set all items in dictionary to one
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: dict_to_one
+
+Convert list of string to dictionary
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: list_string_to_dict
+
+Flatten a list
+^^^^^^^^^^^^^^^^^^^
+.. autofunction:: flatten_list
+
+Close TF session and associated processes
+-----------------------------------------
+.. autofunction:: exit_tensorflow
+
+Open TensorBoard
+----------------
+.. autofunction:: open_tensorboard
+
+Set GPU functions
+-----------------
+.. autofunction:: set_gpu_fraction
diff --git a/docs/modules/visualize.rst b/docs/modules/visualize.rst
new file mode 100644
index 0000000..0bbe028
--- /dev/null
+++ b/docs/modules/visualize.rst
@@ -0,0 +1,76 @@
+API - Visualization
+================================
+
+TensorFlow provides `TensorBoard `_
+to visualize the model, activations etc. Here we provide more functions for data visualization.
+
+.. automodule:: tensorlayer.visualize
+
+.. autosummary::
+
+ read_image
+ read_images
+ save_image
+ save_images
+ draw_boxes_and_labels_to_image
+ draw_mpii_pose_to_image
+ draw_weights
+ CNN2d
+ frame
+ images2d
+ tsne_embedding
+
+
+Save and read images
+----------------------
+
+Read one image
+^^^^^^^^^^^^^^^^^
+.. autofunction:: read_image
+
+Read multiple images
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: read_images
+
+Save one image
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: save_image
+
+Save multiple images
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: save_images
+
+Save image for object detection
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: draw_boxes_and_labels_to_image
+
+
+Save image for pose estimation (MPII)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: draw_mpii_pose_to_image
+
+Visualize model parameters
+------------------------------
+
+Visualize CNN 2d filter
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: CNN2d
+
+Visualize weights
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: draw_weights
+
+Visualize images
+-----------------
+
+Image by matplotlib
+^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: frame
+
+Images by matplotlib
+^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: images2d
+
+Visualize embeddings
+--------------------
+.. autofunction:: tsne_embedding
diff --git a/docs/user/contributing.rst b/docs/user/contributing.rst
new file mode 100644
index 0000000..a83767a
--- /dev/null
+++ b/docs/user/contributing.rst
@@ -0,0 +1,184 @@
+.. _contributing:
+
+===============
+Contributing
+===============
+
+TensorLayer 2.0 is a major ongoing research project in CFCS, Peking University, the first version was established at Imperial College London in 2016. The goal of the project is to develop a compositional language while complex learning systems
+can be built through composition of neural network modules.
+
+Numerous contributors come from various horizons such as: Imperial College London, Tsinghua University, Carnegie Mellon University, Stanford, University of Technology of Compiegne, Google, Microsoft, Bloomberg and etc.
+
+You can easily open a Pull Request (PR) on `GitHub`_, every little step counts and will be credited.
+As an open-source project, we highly welcome and value contributions!
+
+**If you are interested in working with us, please contact us at:** `tensorlayer@gmail.com `_.
+
+.. image:: ../../img/join_slack.png
+ :width: 30 %
+ :align: center
+ :target: https://join.slack.com/t/tensorlayer/shared_invite/enQtMjUyMjczMzU2Njg4LWI0MWU0MDFkOWY2YjQ4YjVhMzI5M2VlZmE4YTNhNGY1NjZhMzUwMmQ2MTc0YWRjMjQzMjdjMTg2MWQ2ZWJhYzc
+
+
+Project Maintainers
+--------------------------
+
+The TensorLayer project was started by `Hao Dong `_ at Imperial College London in June 2016.
+
+For TensorLayer 2.x, it is now actively developing and maintaining by the following people who has more than 50 contributions:
+
+- **Hao Dong** (`@zsdonghao `_) - ``_
+- **Jingqing Zhang** (`@JingqingZ `_) - ``_
+- **Rundi Wu** (`@ChrisWu1997 `_) - ``_
+- **Ruihai Wu** (`@warshallrho `_) - ``_
+
+For TensorLayer 1.x, it was actively developed and maintained by the following people *(in alphabetical order)*:
+
+- **Akara Supratak** (`@akaraspt `_) - ``_
+- **Fangde Liu** (`@fangde `_) - ``_
+- **Guo Li** (`@lgarithm `_) - ``_
+- **Hao Dong** (`@zsdonghao `_) - ``_
+- **Jonathan Dekhtiar** (`@DEKHTIARJonathan `_) - ``_
+- **Luo Mai** (`@luomai `_) - ``_
+- **Simiao Yu** (`@nebulaV `_) - ``_
+
+Numerous other contributors can be found in the `Github Contribution Graph `_.
+
+
+What to contribute
+------------------
+
+Your method and example
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you have a new method or example in terms of Deep learning or Reinforcement learning, you are welcome to contribute.
+
+* Provide your layers or examples, so everyone can use it.
+* Explain how it would work, and link to a scientific paper if applicable.
+* Keep the scope as narrow as possible, to make it easier to implement.
+
+
+Report bugs
+~~~~~~~~~~~
+
+Report bugs at the `GitHub`_, we normally will fix it in 5 hours.
+If you are reporting a bug, please include:
+
+* your TensorLayer, TensorFlow and Python version.
+* steps to reproduce the bug, ideally reduced to a few Python commands.
+* the results you obtain, and the results you expected instead.
+
+If you are unsure whether the behavior you experience is a bug, or if you are
+unsure whether it is related to TensorLayer or TensorFlow, please just ask on `our
+mailing list`_ first.
+
+
+Fix bugs
+~~~~~~~~
+
+Look through the GitHub issues for bug reports. Anything tagged with "bug" is
+open to whoever wants to implement it. If you discover a bug in TensorLayer you can
+fix yourself, by all means feel free to just implement a fix and not report it
+first.
+
+
+Write documentation
+~~~~~~~~~~~~~~~~~~~
+
+Whenever you find something not explained well, misleading, glossed over or
+just wrong, please update it! The *Edit on GitHub* link on the top right of
+every documentation page and the *[source]* link for every documented entity
+in the API reference will help you to quickly locate the origin of any text.
+
+
+
+How to contribute
+-----------------
+
+Edit on GitHub
+~~~~~~~~~~~~~~
+
+As a very easy way of just fixing issues in the documentation, use the *Edit
+on GitHub* link on the top right of a documentation page or the *[source]* link
+of an entity in the API reference to open the corresponding source file in
+GitHub, then click the *Edit this file* link to edit the file in your browser
+and send us a Pull Request. All you need for this is a free GitHub account.
+
+For any more substantial changes, please follow the steps below to setup
+TensorLayer for development.
+
+
+Documentation
+~~~~~~~~~~~~~
+
+The documentation is generated with `Sphinx
+`_. To build it locally, run the
+following commands:
+
+.. code:: bash
+
+ pip install Sphinx
+ sphinx-quickstart
+
+ cd docs
+ make html
+
+If you want to re-generate the whole docs, run the following commands:
+
+.. code :: bash
+
+ cd docs
+ make clean
+ make html
+
+
+To write the docs, we recommend you to install `Local RTD VM `_.
+
+
+
+
+Afterwards, open ``docs/_build/html/index.html`` to view the documentation as
+it would appear on `readthedocs `_. If you
+changed a lot and seem to get misleading error messages or warnings, run
+``make clean html`` to force Sphinx to recreate all files from scratch.
+
+When writing docstrings, follow existing documentation as much as possible to
+ensure consistency throughout the library. For additional information on the
+syntax and conventions used, please refer to the following documents:
+
+* `reStructuredText Primer `_
+* `Sphinx reST markup constructs `_
+* `A Guide to NumPy/SciPy Documentation `_
+
+
+Testing
+~~~~~~~
+
+TensorLayer has a code coverage of 100%, which has proven very helpful in the past,
+but also creates some duties:
+
+* Whenever you change any code, you should test whether it breaks existing
+ features by just running the test scripts.
+* Every bug you fix indicates a missing test case, so a proposed bug fix should
+ come with a new test that fails without your fix.
+
+
+Sending Pull Requests
+~~~~~~~~~~~~~~~~~~~~~
+
+When you're satisfied with your addition, the tests pass and the documentation
+looks good without any markup errors, commit your changes to a new branch, push
+that branch to your fork and send us a Pull Request via GitHub's web interface.
+
+All these steps are nicely explained on GitHub:
+https://guides.github.com/introduction/flow/
+
+When filing your Pull Request, please include a description of what it does, to
+help us reviewing it. If it is fixing an open issue, say, issue #123, add
+*Fixes #123*, *Resolves #123* or *Closes #123* to the description text, so
+GitHub will close it when your request is merged.
+
+
+.. _Release: https://github.com/tensorlayer/tensorlayer/releases
+.. _GitHub: https://github.com/tensorlayer/tensorlayer
+.. _our mailing list: hao.dong11@imperial.ac.uk
diff --git a/docs/user/examples.rst b/docs/user/examples.rst
new file mode 100644
index 0000000..91971c0
--- /dev/null
+++ b/docs/user/examples.rst
@@ -0,0 +1,121 @@
+.. _example:
+
+============
+Examples
+============
+
+We list some examples here, but more tutorials and applications can be found in `Github examples `__ and `Awesome-TensorLayer `_.
+
+Basics
+============
+
+ - Multi-layer perceptron (MNIST), simple usage. Classification task, see `tutorial_mnist_simple.py `__.
+ - Multi-layer perceptron (MNIST), dynamic model. Classification with dropout using iterator, see `tutorial_mnist_mlp_dynamic.py method2 `__.
+ - Multi-layer perceptron (MNIST), static model. Classification with dropout using iterator, see `tutorial_mnist_mlp_static.py `__.
+ - Convolutional Network (CIFAR-10). Classification task, see `tutorial_cifar10_cnn_static.py `_.
+ - TensorFlow dataset API for object detection see `here `__.
+ - Data augmentation with TFRecord. Effective way to load and pre-process data, see `tutorial_tfrecord*.py `__ and `tutorial_cifar10_tfrecord.py `__.
+ - Data augmentation with TensorLayer. See `tutorial_fast_affine_transform.py `__ (for quick test only).
+
+Pretrained Models
+==================
+
+ - VGG 16 (ImageNet). Classification task, see `tutorial_models_vgg16 `__.
+ - VGG 19 (ImageNet). Classification task, see `tutorial_models_vgg19.py `__.
+ - SqueezeNet (ImageNet). Model compression, see `tutorial_models_squeezenetv1.py `__.
+ - MobileNet (ImageNet). Model compression, see `tutorial_models_mobilenetv1.py `__.
+ - All pretrained models in `pretrained-models `__.
+
+Vision
+==================
+
+ - Arbitrary Style Transfer in Real-time with Adaptive Instance Normalization, see `examples `__.
+ - ArcFace: Additive Angular Margin Loss for Deep Face Recognition, see `InsignFace `__.
+ - BinaryNet. Model compression, see `mnist `__ `cifar10 `__.
+ - Ternary Weight Network. Model compression, see `mnist `__ `cifar10 `__.
+ - DoReFa-Net. Model compression, see `mnist `__ `cifar10 `__.
+ - QuanCNN. Model compression, sees `mnist `__ `cifar10 `__.
+ - Wide ResNet (CIFAR) by `ritchieng `__.
+ - `Spatial Transformer Networks `__ by `zsdonghao `__.
+ - `U-Net for brain tumor segmentation `__ by `zsdonghao `__.
+ - Variational Autoencoder (VAE) for (CelebA) by `yzwxx `__.
+ - Variational Autoencoder (VAE) for (MNIST) by `BUPTLdy `__.
+ - Image Captioning - Reimplementation of Google's `im2txt `__ by `zsdonghao `__.
+
+Adversarial Learning
+========================
+ - DCGAN (CelebA). Generating images by `Deep Convolutional Generative Adversarial Networks `__ by `zsdonghao `__.
+ - `Generative Adversarial Text to Image Synthesis `__ by `zsdonghao `__.
+ - `Unsupervised Image to Image Translation with Generative Adversarial Networks `__ by `zsdonghao `__.
+ - `Improved CycleGAN `__ with resize-convolution by `luoxier `__.
+ - `Super Resolution GAN `__ by `zsdonghao `__.
+ - `BEGAN: Boundary Equilibrium Generative Adversarial Networks `__ by `2wins `__.
+ - `DAGAN: Fast Compressed Sensing MRI Reconstruction `__ by `nebulaV `__.
+
+Natural Language Processing
+==============================
+
+ - Recurrent Neural Network (LSTM). Apply multiple LSTM to PTB dataset for language modeling, see `tutorial_ptb_lstm_state_is_tuple.py `__.
+ - Word Embedding (Word2vec). Train a word embedding matrix, see `tutorial_word2vec_basic.py `__.
+ - Restore Embedding matrix. Restore a pre-train embedding matrix, see `tutorial_generate_text.py `__.
+ - Text Generation. Generates new text scripts, using LSTM network, see `tutorial_generate_text.py `__.
+ - Chinese Text Anti-Spam by `pakrchen `__.
+ - `Chatbot in 200 lines of code `__ for `Seq2Seq `__.
+ - FastText Sentence Classification (IMDB), see `tutorial_imdb_fasttext.py `__ by `tomtung `__.
+
+Reinforcement Learning
+==============================
+
+ - Policy Gradient / Network (Atari Ping Pong), see `tutorial_atari_pong.py `__.
+ - Deep Q-Network (Frozen lake), see `tutorial_frozenlake_dqn.py `__.
+ - Q-Table learning algorithm (Frozen lake), see `tutorial_frozenlake_q_table.py `__.
+ - Asynchronous Policy Gradient using TensorDB (Atari Ping Pong) by `nebulaV `__.
+ - AC for discrete action space (Cartpole), see `tutorial_cartpole_ac.py `__.
+ - A3C for continuous action space (Bipedal Walker), see `tutorial_bipedalwalker_a3c*.py `__.
+ - `DAGGER `__ for (`Gym Torcs `__) by `zsdonghao `__.
+ - `TRPO `__ for continuous and discrete action space by `jjkke88 `__.
+
+Miscellaneous
+=================
+
+- `Sipeed `__ : Run TensorLayer on AI Chips
+
+..
+ - TensorDB by `fangde `__ see `tl_paper `__.
+ - A simple web service - `TensorFlask `__ by `JoelKronander `__.
+
+..
+ Applications
+ =============
+
+ There are some good applications implemented by TensorLayer.
+ You may able to find some useful examples for your project.
+ If you want to share your application, please contact tensorlayer@gmail.com.
+
+ 1D CNN + LSTM for Biosignal
+ ---------------------------------
+
+ Author : `Akara Supratak `__
+
+ Introduction
+ ^^^^^^^^^^^^
+
+ Implementation
+ ^^^^^^^^^^^^^^
+
+ Citation
+ ^^^^^^^^
+
+
+
+
+
+.. _GitHub: https://github.com/tensorlayer/tensorlayer
+.. _Deeplearning Tutorial: http://deeplearning.stanford.edu/tutorial/
+.. _Convolutional Neural Networks for Visual Recognition: http://cs231n.github.io/
+.. _Neural Networks and Deep Learning: http://neuralnetworksanddeeplearning.com/
+.. _TensorFlow tutorial: https://www.tensorflow.org/versions/r0.9/tutorials/index.html
+.. _Understand Deep Reinforcement Learning: http://karpathy.github.io/2016/05/31/rl/
+.. _Understand Recurrent Neural Network: http://karpathy.github.io/2015/05/21/rnn-effectiveness/
+.. _Understand LSTM Network: http://colah.github.io/posts/2015-08-Understanding-LSTMs/
+.. _Word Representations: http://colah.github.io/posts/2014-07-NLP-RNNs-Representations/
diff --git a/docs/user/faq.rst b/docs/user/faq.rst
new file mode 100644
index 0000000..0c05033
--- /dev/null
+++ b/docs/user/faq.rst
@@ -0,0 +1,79 @@
+.. _faq:
+
+============
+FAQ
+============
+
+
+How to effectively learn TensorLayer
+=====================================
+
+No matter what stage you are in, we recommend you to spend just 10 minutes to
+read the source code of TensorLayer and the `Understand layer / Your layer `__
+in this website, you will find the abstract methods are very simple for everyone.
+Reading the source codes helps you to better understand TensorFlow and allows
+you to implement your own methods easily. For discussion, we recommend
+`Gitter `__,
+`Help Wanted Issues `__,
+`QQ group `__
+and `Wechat group `__.
+
+Beginner
+-----------
+For people who new to deep learning, the contributors provided a number of tutorials in this website, these tutorials will guide you to understand autoencoder, convolutional neural network, recurrent neural network, word embedding and deep reinforcement learning and etc. If your already understand the basic of deep learning, we recommend you to skip the tutorials and read the example codes on `Github `__ , then implement an example from scratch.
+
+Engineer
+------------
+For people from industry, the contributors provided mass format-consistent examples covering computer vision, natural language processing and reinforcement learning. Besides, there are also many TensorFlow users already implemented product-level examples including image captioning, semantic/instance segmentation, machine translation, chatbot and etc., which can be found online.
+It is worth noting that a wrapper especially for computer vision `Tf-Slim `__ can be connected with TensorLayer seamlessly.
+Therefore, you may able to find the examples that can be used in your project.
+
+Researcher
+-------------
+For people from academia, TensorLayer was originally developed by PhD students who facing issues with other libraries on implement novel algorithm. Installing TensorLayer in editable mode is recommended, so you can extend your methods in TensorLayer.
+For research related to image processing such as image captioning, visual QA and etc., you may find it is very helpful to use the existing `Tf-Slim pre-trained models `__ with TensorLayer (a specially layer for connecting Tf-Slim is provided).
+
+
+Install Master Version
+========================
+
+To use all new features of TensorLayer, you need to install the master version from Github.
+Before that, you need to make sure you already installed git.
+
+.. code-block:: bash
+
+ [stable version] pip install tensorlayer
+ [master version] pip install git+https://github.com/tensorlayer/tensorlayer.git
+
+Editable Mode
+===============
+
+- 1. Download the TensorLayer folder from Github.
+- 2. Before editing the TensorLayer ``.py`` file.
+
+ - If your script and TensorLayer folder are in the same folder, when you edit the ``.py`` inside TensorLayer folder, your script can access the new features.
+ - If your script and TensorLayer folder are not in the same folder, you need to run the following command in the folder contains ``setup.py`` before you edit ``.py`` inside TensorLayer folder.
+
+ .. code-block:: bash
+
+ pip install -e .
+
+
+Load Model
+===========
+
+Note that, the ``tl.files.load_npz()`` can only able to load the npz model saved by ``tl.files.save_npz()``.
+If you have a model want to load into your TensorLayer network, you can first assign your parameters into a list in order,
+then use ``tl.files.assign_params()`` to load the parameters into your TensorLayer model.
+
+
+
+.. _GitHub: https://github.com/tensorlayer/tensorlayer
+.. _Deeplearning Tutorial: http://deeplearning.stanford.edu/tutorial/
+.. _Convolutional Neural Networks for Visual Recognition: http://cs231n.github.io/
+.. _Neural Networks and Deep Learning: http://neuralnetworksanddeeplearning.com/
+.. _TensorFlow tutorial: https://www.tensorflow.org/versions/r0.9/tutorials/index.html
+.. _Understand Deep Reinforcement Learning: http://karpathy.github.io/2016/05/31/rl/
+.. _Understand Recurrent Neural Network: http://karpathy.github.io/2015/05/21/rnn-effectiveness/
+.. _Understand LSTM Network: http://colah.github.io/posts/2015-08-Understanding-LSTMs/
+.. _Word Representations: http://colah.github.io/posts/2014-07-NLP-RNNs-Representations/
diff --git a/docs/user/get_involved.rst b/docs/user/get_involved.rst
new file mode 100644
index 0000000..3fe4131
--- /dev/null
+++ b/docs/user/get_involved.rst
@@ -0,0 +1,76 @@
+
+
+=========================
+Get Involved in Research
+=========================
+
+Ph.D. Postition @ PKU
+=============================================================
+
+
+Hi, I am `Hao Dong `__, the founder of this project and a new faculty member in EECS, Peking University. I now have a few Ph.D. positions per year open for international students who would like to study AI. If you or your friends are interested in it, feel free to contact me.
+PKU is a top 30 university in the global ranking. The application is competitive, apply early is recommended. For the application of next year, please note that the DDL of Chinese Government Scholarship is in the end of each year, please check this `link `__ for more details.
+
+My homepage: `https://zsdonghao.github.io `__
+
+Contact: hao.dong [AT] pku.edu.cn
+
+
+
+Faculty Postition @ PKU
+=============================================================
+
+The Center on Frontiers of Computing Studies (CFCS), Peking University (PKU), China, is a university new initiative co-founded by Professors John Hopcroft (Turing Awardee) and Wen Gao (CAE, ACM/IEEE Fellow). The center aims at developing the excellence on two fronts: research and education. On the research front, the center will provide a world-class research environment, where innovation and impactful research is the central aim, measured by professional reputation among world scholars, not by counting the number of publications and research funding. On the education front, the center deeply involves in the Turing Class, an elite undergraduate program that draws the cream of the crop from the PKU undergraduate talent pool. New curriculum and pedagogy are designed and practiced in this program, with the aim to cultivate a new generation of computer scientist/engineers that are solid in both theories and practices.
+
+**Positions and Qualification**
+
+The center invites applications for tenured/tenure-track faculty positions. We are seeking applicants from all areas of Computer Science, spanning theoretical foundations, systems, software, and applications, with special interests in artificial intelligence and machine learning. We are especially interested in applicants conducting research at the frontiers of Computer Science with other disciplines, such as data sciences, engineering, as well as mathematical, medical, physical, and social sciences.
+
+Applicants are expected to have completed (or be completing) a Ph.D., have demonstrated the ability to pursue a program of research, and have a strong commitment to undergraduate and graduate teaching. A successful candidate will be expected to teach one to two courses at the undergraduate and graduate levels in each semester, and to build and lead a team of undergraduate and graduate students in innovative research.
+
+We are also seeking qualified candidates for postdoctoral positions. Candidates should have a Ph.D. in a relevant discipline or expect a Ph. D within a year, with a substantive record of research accomplishments, and the ability to work collaboratively with faculty members in the center.
+
+**To Apply**
+
+Applicants should send a full curriculum vitae; copies of 3-5 key publications; 3-5 names and contact information of references; and a statement of research and teaching to: CFCS_recruiting[at]pku[dot]edu[dot]cn . To expedite the process, please arrange to have the reference letters sent directly to the above email address.
+
+Application for a postdoctoral position should include a curriculum vita, brief statement of research, and three to five names and contact information of recommendation, and can be directly addressed to an individual faculty member.
+
+We conduct review of applications monthly, immediately upon the recipient of all application materials at the beginning of each month. However, it is highly recommended that applicants submit complete applications sooner than later, as the positions are to be filled quickly.
+
+
+Postdoc Postition @ ICL
+==================================================
+
+Data science is therefore by nature at the core of all modern transdisciplinary scientific activities, as it involves the whole life cycle of data, from acquisition and exploration to analysis and communication of the results. Data science is not only concerned with the tools and methods to obtain, manage and analyse data: it is also about extracting value from data and translating it from asset to product.
+
+Launched on 1st April 2014, the Data Science Institute (DSI) at Imperial College London aims to enhance Imperial's excellence in data-driven research across its faculties by fulfilling the following objectives.
+
+The Data Science Institute is housed in purpose built facilities in the heart of the Imperial College campus in South Kensington. Such a central location provides excellent access to collabroators across the College and across London.
+
+ - To act as a focal point for coordinating data science research at Imperial College by facilitating access to funding, engaging with global partners, and stimulating cross-disciplinary collaboration.
+ - To develop data management and analysis technologies and services for supporting data driven research in the College.
+ - To promote the training and education of the new generation of data scientist by developing and coordinating new degree courses, and conducting public outreach programmes on data science.
+ - To advise College on data strategy and policy by providing world-class data science expertise.
+ - To enable the translation of data science innovation by close collaboration with industry and supporting commercialization.
+
+If you are interested in working with us, please check our
+`vacancies `__
+and other ways to
+`get involved `__
+, or feel free to
+`contact us `__.
+
+Software Engineer @ SurgicalAI.cn
+=============================================================
+SurgicalAI is a startup founded by the data scientists and surgical robot experts from Imperial College. Our objective is AI democratise Surgery. By combining 5G, AI and Cloud Computing, SurgicalAI is building a platform enable junor surgeons to perfom complex procedures. As one of the most impactful startup, SurgicalAI is supported by Nvidia, AWS and top surgeons around the world.
+
+Currently based in Hangzhou, China, we are building digital solution for cardiac surgery like TAVR, LAA and Orthopedidcs like TKA and UNA. A demo can be found at here
+
+We are activly looking for experts in robotic navigation, computer graphics and medical image analysis experts to join us, building digitalized surgical service platform for the aging world.
+
+Home Page: http://www.surgicalai.cn
+
+Demo Page: http://demo5g.surgicalai.cn
+
+Contact: liufangde@surgicalai.cn
diff --git a/docs/user/get_start_advance.rst b/docs/user/get_start_advance.rst
new file mode 100644
index 0000000..db3441c
--- /dev/null
+++ b/docs/user/get_start_advance.rst
@@ -0,0 +1,217 @@
+.. _getstartadvance:
+
+==================
+Advanced features
+==================
+
+
+Customizing layer
+==================
+
+Layers with weights
+----------------------
+
+The fully-connected layer is `a = f(x*W+b)`, the most simple implementation is as follow, which can only support static model.
+
+.. code-block:: python
+
+ class Dense(Layer):
+ """The :class:`Dense` class is a fully connected layer.
+
+ Parameters
+ ----------
+ n_units : int
+ The number of units of this layer.
+ act : activation function
+ The activation function of this layer.
+ name : None or str
+ A unique layer name. If None, a unique name will be automatically generated.
+ """
+
+ def __init__(
+ self,
+ n_units, # the number of units/channels of this layer
+ act=None, # None: no activation, tf.nn.relu or 'relu': ReLU ...
+ name=None, # the name of this layer (optional)
+ ):
+ super(Dense, self).__init__(name, act=act) # auto naming, dense_1, dense_2 ...
+ self.n_units = n_units
+
+ def build(self, inputs_shape): # initialize the model weights here
+ shape = [inputs_shape[1], self.n_units]
+ self.W = self._get_weights("weights", shape=tuple(shape), init=self.W_init)
+ self.b = self._get_weights("biases", shape=(self.n_units, ), init=self.b_init)
+
+ def forward(self, inputs): # call function
+ z = tf.matmul(inputs, self.W) + self.b
+ if self.act: # is not None
+ z = self.act(z)
+ return z
+
+The full implementation is as follow, which supports both static and dynamic models and allows users to control whether to use the bias, how to initialize the weight values.
+
+.. code-block:: python
+
+ class Dense(Layer):
+ """The :class:`Dense` class is a fully connected layer.
+
+ Parameters
+ ----------
+ n_units : int
+ The number of units of this layer.
+ act : activation function
+ The activation function of this layer.
+ W_init : initializer
+ The initializer for the weight matrix.
+ b_init : initializer or None
+ The initializer for the bias vector. If None, skip biases.
+ in_channels: int
+ The number of channels of the previous layer.
+ If None, it will be automatically detected when the layer is forwarded for the first time.
+ name : None or str
+ A unique layer name. If None, a unique name will be automatically generated.
+ """
+
+ def __init__(
+ self,
+ n_units,
+ act=None,
+ W_init=tl.initializers.truncated_normal(stddev=0.1),
+ b_init=tl.initializers.constant(value=0.0),
+ in_channels=None, # the number of units/channels of the previous layer
+ name=None,
+ ):
+ # we feed activation function to the base layer, `None` denotes identity function
+ # string (e.g., relu, sigmoid) will be converted into function.
+ super(Dense, self).__init__(name, act=act)
+
+ self.n_units = n_units
+ self.W_init = W_init
+ self.b_init = b_init
+ self.in_channels = in_channels
+
+ # in dynamic model, the number of input channel is given, we initialize the weights here
+ if self.in_channels is not None:
+ self.build(self.in_channels)
+ self._built = True
+
+ logging.info(
+ "Dense %s: %d %s" %
+ (self.name, self.n_units, self.act.__name__ if self.act is not None else 'No Activation')
+ )
+
+ def __repr__(self): # optional, for printing information
+ actstr = self.act.__name__ if self.act is not None else 'No Activation'
+ s = ('{classname}(n_units={n_units}, ' + actstr)
+ if self.in_channels is not None:
+ s += ', in_channels=\'{in_channels}\''
+ if self.name is not None:
+ s += ', name=\'{name}\''
+ s += ')'
+ return s.format(classname=self.__class__.__name__, **self.__dict__)
+
+ def build(self, inputs_shape): # initialize the model weights here
+ if self.in_channels: # if the number of input channel is given, use it
+ shape = [self.in_channels, self.n_units]
+ else: # otherwise, get it from static model
+ self.in_channels = inputs_shape[1]
+ shape = [inputs_shape[1], self.n_units]
+ self.W = self._get_weights("weights", shape=tuple(shape), init=self.W_init)
+ if self.b_init: # if b_init is None, no bias is applied
+ self.b = self._get_weights("biases", shape=(self.n_units, ), init=self.b_init)
+
+ def forward(self, inputs):
+ z = tf.matmul(inputs, self.W)
+ if self.b_init:
+ z = tf.add(z, self.b)
+ if self.act:
+ z = self.act(z)
+ return z
+
+
+Layers with train/test modes
+------------------------------
+
+We use Dropout as an example here:
+
+.. code-block:: python
+
+ class Dropout(Layer):
+ """
+ The :class:`Dropout` class is a noise layer which randomly set some
+ activations to zero according to a keeping probability.
+ Parameters
+ ----------
+ keep : float
+ The keeping probability.
+ The lower the probability it is, the more activations are set to zero.
+ name : None or str
+ A unique layer name.
+ """
+
+ def __init__(self, keep, name=None):
+ super(Dropout, self).__init__(name)
+ self.keep = keep
+
+ self.build()
+ self._built = True
+
+ logging.info("Dropout %s: keep: %f " % (self.name, self.keep))
+
+ def build(self, inputs_shape=None):
+ pass # no weights in dropout layer
+
+ def forward(self, inputs):
+ if self.is_train: # this attribute is changed by Model.train() and Model.eval() described above
+ outputs = tf.nn.dropout(inputs, rate=1 - (self.keep), name=self.name)
+ else:
+ outputs = inputs
+ return outputs
+
+Pre-trained CNN
+================
+
+Get entire CNN
+---------------
+
+.. code-block:: python
+
+ import tensorflow as tf
+ import tensorlayer as tl
+ import numpy as np
+ from tensorlayer.models.imagenet_classes import class_names
+
+ vgg = tl.models.vgg16(pretrained=True)
+ img = tl.vis.read_image('data/tiger.jpeg')
+ img = tl.prepro.imresize(img, (224, 224)).astype(np.float32) / 255
+ output = vgg(img, is_train=False)
+
+Get a part of CNN
+------------------
+
+.. code-block:: python
+
+ # get VGG without the last layer
+ cnn = tl.models.vgg16(end_with='fc2_relu', mode='static').as_layer()
+ # add one more layer and build a new model
+ ni = tl.layers.Input([None, 224, 224, 3], name="inputs")
+ nn = cnn(ni)
+ nn = tl.layers.Dense(n_units=100, name='out')(nn)
+ model = tl.models.Model(inputs=ni, outputs=nn)
+ # train your own classifier (only update the last layer)
+ train_weights = model.get_layer('out').all_weights
+
+Reuse CNN
+------------------
+
+.. code-block:: python
+
+ # in dynamic model, we can directly use the same model
+ # in static model
+ vgg_layer = tl.models.vgg16().as_layer()
+ ni_1 = tl.layers.Input([None, 224, 224, 3])
+ ni_2 = tl.layers.Input([None, 224, 224, 3])
+ a_1 = vgg_layer(ni_1)
+ a_2 = vgg_layer(ni_2)
+ M = Model(inputs=[ni_1, ni_2], outputs=[a_1, a_2])
+
diff --git a/docs/user/get_start_model.rst b/docs/user/get_start_model.rst
new file mode 100644
index 0000000..2337a7d
--- /dev/null
+++ b/docs/user/get_start_model.rst
@@ -0,0 +1,249 @@
+.. _getstartmodel:
+
+===============
+Define a model
+===============
+
+TensorLayer provides two ways to define a model.
+Static model allows you to build model in a fluent way while dynamic model allows you to fully control the forward process.
+
+Static model
+===============
+
+.. code-block:: python
+
+ import tensorflow as tf
+ from tensorlayer.layers import Input, Dropout, Dense
+ from tensorlayer.models import Model
+
+ def get_model(inputs_shape):
+ ni = Input(inputs_shape)
+ nn = Dropout(keep=0.8)(ni)
+ nn = Dense(n_units=800, act=tf.nn.relu, name="dense1")(nn) # “name" is optional
+ nn = Dropout(keep=0.8)(nn)
+ nn = Dense(n_units=800, act=tf.nn.relu)(nn)
+ nn = Dropout(keep=0.8)(nn)
+ nn = Dense(n_units=10, act=None)(nn)
+ M = Model(inputs=ni, outputs=nn, name="mlp") # “name" is optional
+ return M
+
+ MLP = get_model([None, 784])
+ MLP.eval()
+ outputs = MLP(data)
+
+Dynamic model
+=======================
+
+
+In this case, you need to manually input the output shape of the previous layer to the new layer.
+
+.. code-block:: python
+
+ class CustomModel(Model):
+
+ def __init__(self):
+ super(CustomModel, self).__init__()
+
+ self.dropout1 = Dropout(keep=0.8)
+ self.dense1 = Dense(n_units=800, act=tf.nn.relu, in_channels=784)
+ self.dropout2 = Dropout(keep=0.8)
+ self.dense2 = Dense(n_units=800, act=tf.nn.relu, in_channels=800)
+ self.dropout3 = Dropout(keep=0.8)
+ self.dense3 = Dense(n_units=10, act=None, in_channels=800)
+
+ def forward(self, x, foo=False):
+ z = self.dropout1(x)
+ z = self.dense1(z)
+ z = self.dropout2(z)
+ z = self.dense2(z)
+ z = self.dropout3(z)
+ out = self.dense3(z)
+ if foo:
+ out = tf.nn.softmax(out)
+ return out
+
+ MLP = CustomModel()
+ MLP.eval()
+ outputs = MLP(data, foo=True) # controls the forward here
+ outputs = MLP(data, foo=False)
+
+
+Switching train/test modes
+=============================
+
+.. code-block:: python
+
+ # method 1: switch before forward
+ Model.train() # enable dropout, batch norm moving avg ...
+ output = Model(train_data)
+ ... # training code here
+ Model.eval() # disable dropout, batch norm moving avg ...
+ output = Model(test_data)
+ ... # testing code here
+
+ # method 2: switch while forward
+ output = Model(train_data, is_train=True)
+ output = Model(test_data, is_train=False)
+
+Reuse weights
+=======================
+
+For static model, call the layer multiple time in model creation
+
+.. code-block:: python
+
+ # create siamese network
+
+ def create_base_network(input_shape):
+ '''Base network to be shared (eq. to feature extraction).
+ '''
+ input = Input(shape=input_shape)
+ x = Flatten()(input)
+ x = Dense(128, act=tf.nn.relu)(x)
+ x = Dropout(0.9)(x)
+ x = Dense(128, act=tf.nn.relu)(x)
+ x = Dropout(0.9)(x)
+ x = Dense(128, act=tf.nn.relu)(x)
+ return Model(input, x)
+
+
+ def get_siamese_network(input_shape):
+ """Create siamese network with shared base network as layer
+ """
+ base_layer = create_base_network(input_shape).as_layer() # convert model as layer
+
+ ni_1 = Input(input_shape)
+ ni_2 = Input(input_shape)
+ nn_1 = base_layer(ni_1) # call base_layer twice
+ nn_2 = base_layer(ni_2)
+ return Model(inputs=[ni_1, ni_2], outputs=[nn_1, nn_2])
+
+ siamese_net = get_siamese_network([None, 784])
+
+For dynamic model, call the layer multiple time in forward function
+
+.. code-block:: python
+
+ class MyModel(Model):
+ def __init__(self):
+ super(MyModel, self).__init__()
+ self.dense_shared = Dense(n_units=800, act=tf.nn.relu, in_channels=784)
+ self.dense1 = Dense(n_units=10, act=tf.nn.relu, in_channels=800)
+ self.dense2 = Dense(n_units=10, act=tf.nn.relu, in_channels=800)
+ self.cat = Concat()
+
+ def forward(self, x):
+ x1 = self.dense_shared(x) # call dense_shared twice
+ x2 = self.dense_shared(x)
+ x1 = self.dense1(x1)
+ x2 = self.dense2(x2)
+ out = self.cat([x1, x2])
+ return out
+
+ model = MyModel()
+
+Print model information
+=======================
+
+.. code-block:: python
+
+ print(MLP) # simply call print function
+
+ # Model(
+ # (_inputlayer): Input(shape=[None, 784], name='_inputlayer')
+ # (dropout): Dropout(keep=0.8, name='dropout')
+ # (dense): Dense(n_units=800, relu, in_channels='784', name='dense')
+ # (dropout_1): Dropout(keep=0.8, name='dropout_1')
+ # (dense_1): Dense(n_units=800, relu, in_channels='800', name='dense_1')
+ # (dropout_2): Dropout(keep=0.8, name='dropout_2')
+ # (dense_2): Dense(n_units=10, None, in_channels='800', name='dense_2')
+ # )
+
+ import pprint
+ pprint.pprint(MLP.config) # print the model architecture
+ # {'inputs': '_inputlayer_1_node_0',
+ # 'model_architecture': [{'args': {'dtype': tf.float32,
+ # 'layer_type': 'normal',
+ # 'name': '_inputlayer_1',
+ # 'shape': [None, 784]},
+ # 'class': '_InputLayer',
+ # 'prev_layer': None},
+ # {'args': {'keep': 0.8,
+ # 'layer_type': 'normal',
+ # 'name': 'dropout_1'},
+ # 'class': 'Dropout',
+ # 'prev_layer': ['_inputlayer_1_node_0']},
+ # {'args': {'act': 'relu',
+ # 'layer_type': 'normal',
+ # 'n_units': 800,
+ # 'name': 'dense_1'},
+ # 'class': 'Dense',
+ # 'prev_layer': ['dropout_1_node_0']},
+ # {'args': {'keep': 0.8,
+ # 'layer_type': 'normal',
+ # 'name': 'dropout_2'},
+ # 'class': 'Dropout',
+ # 'prev_layer': ['dense_1_node_0']},
+ # {'args': {'act': 'relu',
+ # 'layer_type': 'normal',
+ # 'n_units': 800,
+ # 'name': 'dense_2'},
+ # 'class': 'Dense',
+ # 'prev_layer': ['dropout_2_node_0']},
+ # {'args': {'keep': 0.8,
+ # 'layer_type': 'normal',
+ # 'name': 'dropout_3'},
+ # 'class': 'Dropout',
+ # 'prev_layer': ['dense_2_node_0']},
+ # {'args': {'act': None,
+ # 'layer_type': 'normal',
+ # 'n_units': 10,
+ # 'name': 'dense_3'},
+ # 'class': 'Dense',
+ # 'prev_layer': ['dropout_3_node_0']}],
+ # 'name': 'mlp',
+ # 'outputs': 'dense_3_node_0',
+ # 'version_info': {'backend': 'tensorflow',
+ # 'backend_version': '2.0.0-alpha0',
+ # 'save_date': None,
+ # 'tensorlayer_version': '2.1.0',
+ # 'training_device': 'gpu'}}
+
+Get specific weights
+=======================
+
+We can get the specific weights by indexing or naming.
+
+.. code-block:: python
+
+ # indexing
+ all_weights = MLP.all_weights
+ some_weights = MLP.all_weights[1:3]
+
+ # naming
+ some_weights = MLP.get_layer('dense1').all_weights
+
+
+Save and restore model
+=======================
+
+We provide two ways to save and restore models
+
+
+Save weights only
+------------------
+
+.. code-block:: python
+
+ MLP.save_weights('model_weights.h5') # by default, file will be in hdf5 format
+ MLP.load_weights('model_weights.h5')
+
+Save model architecture and weights (optional)
+-----------------------------------------------
+
+.. code-block:: python
+
+ # When using Model.load(), there is no need to reimplement or declare the architecture of the model explicitly in code
+ MLP.save('model.h5', save_weights=True)
+ MLP = Model.load('model.h5', load_weights=True)
+
diff --git a/docs/user/installation.rst b/docs/user/installation.rst
new file mode 100644
index 0000000..bb86054
--- /dev/null
+++ b/docs/user/installation.rst
@@ -0,0 +1,210 @@
+.. _installation:
+
+============
+Installation
+============
+
+TensorLayer has some prerequisites that need to be installed first, including
+`TensorFlow`_ , numpy and matplotlib. For GPU
+support CUDA and cuDNN are required.
+
+If you run into any trouble, please check the `TensorFlow installation
+instructions `_
+which cover installing the TensorFlow for a range of operating systems including
+Mac OX, Linux and Windows, or ask for help on `tensorlayer@gmail.com `_
+or `FAQ `_.
+
+
+Install TensorFlow
+=========================
+
+.. code-block:: bash
+
+ pip3 install tensorflow-gpu==2.0.0-beta1 # specific version (YOU SHOULD INSTALL THIS ONE NOW)
+ pip3 install tensorflow-gpu # GPU version
+ pip3 install tensorflow # CPU version
+
+The installation instructions of TensorFlow are written to be very detailed on `TensorFlow`_ website.
+However, there are something need to be considered. For example, `TensorFlow`_ officially supports GPU acceleration for Linux, Mac OX and Windows at present. For ARM processor architecture, you need to install TensorFlow from source.
+
+Install TensorLayer
+=========================
+
+For stable version:
+
+.. code-block:: bash
+
+ pip3 install tensorlayer
+
+For latest version, please install from Github.
+
+.. code-block:: bash
+
+ pip3 install git+https://github.com/tensorlayer/tensorlayer.git
+ or
+ pip3 install https://github.com/tensorlayer/tensorlayer/archive/master.zip
+
+For developers, you should clone the folder to your local machine and put it along with your project scripts.
+
+.. code-block:: bash
+
+ git clone https://github.com/tensorlayer/tensorlayer.git
+
+
+Alternatively, you can build from the source.
+
+.. code-block:: bash
+
+ # First clone the repository and change the current directory to the newly cloned repository
+ git clone https://github.com/tensorlayer/tensorlayer.git
+ cd tensorlayer
+
+ # Install virtualenv if necessary
+ pip install virtualenv
+ # Then create a virtualenv called `venv`
+ virtualenv venv
+
+ # Activate the virtualenv
+
+ ## Linux:
+ source venv/bin/activate
+
+ ## Windows:
+ venv\Scripts\activate.bat
+
+ # basic installation
+ pip install .
+
+ # ============= IF TENSORFLOW IS NOT ALREADY INSTALLED ============= #
+
+ # for a machine **without** an NVIDIA GPU
+ pip install -e ".[all_cpu_dev]"
+
+ # for a machine **with** an NVIDIA GPU
+ pip install -e ".[all_gpu_dev]"
+
+If you want install TensorLayer 1.X, the simplest way to install TensorLayer 1.X is as follow. It will also install the numpy and matplotlib automatically.
+
+.. code-block:: bash
+
+ [stable version] pip install tensorlayer==1.x.x
+
+However, if you want to modify or extend TensorLayer 1.X, you can download the repository from
+`Github`_ and install it as follow.
+
+.. code-block:: bash
+
+ cd to the root of the git tree
+ pip install -e .
+
+This command will run the ``setup.py`` to install TensorLayer. The ``-e`` reflects
+editable, then you can edit the source code in ``tensorlayer`` folder, and ``import`` the edited
+TensorLayer.
+
+
+GPU support
+==========================
+
+Thanks to NVIDIA supports, training a fully connected network on a
+GPU, which may be 10 to 20 times faster than training them on a CPU.
+For convolutional network, may have 50 times faster.
+This requires an NVIDIA GPU with CUDA and cuDNN support.
+
+
+CUDA
+----
+
+The TensorFlow website also teach how to install the CUDA and cuDNN, please see
+`TensorFlow GPU Support `_.
+
+Download and install the latest CUDA is available from NVIDIA website:
+
+ - `CUDA download and install `_
+
+
+..
+ After installation, make sure ``/usr/local/cuda/bin`` is in your ``PATH`` (use ``echo #PATH`` to check), and
+ ``nvcc --version`` works. Also ensure ``/usr/local/cuda/lib64`` is in your
+ ``LD_LIBRARY_PATH``, so the CUDA libraries can be found.
+
+If CUDA is set up correctly, the following command should print some GPU information on
+the terminal:
+
+.. code-block:: bash
+
+ python -c "import tensorflow"
+
+
+cuDNN
+--------
+
+Apart from CUDA, NVIDIA also provides a library for common neural network operations that especially
+speeds up Convolutional Neural Networks (CNNs). Again, it can be obtained from
+NVIDIA after registering as a developer (it take a while):
+
+Download and install the latest cuDNN is available from NVIDIA website:
+
+ - `cuDNN download and install `_
+
+
+To install it, copy the ``*.h`` files to ``/usr/local/cuda/include`` and the
+``lib*`` files to ``/usr/local/cuda/lib64``.
+
+.. _TensorFlow: https://www.tensorflow.org/versions/master/get_started/os_setup.html
+.. _GitHub: https://github.com/tensorlayer/tensorlayer
+.. _TensorLayer: https://github.com/tensorlayer/tensorlayer/
+
+
+
+Windows User
+==============
+
+TensorLayer is built on the top of Python-version TensorFlow, so please install Python first.
+Note：We highly recommend installing Anaconda. The lowest version requirements of Python is py35.
+
+`Anaconda download `_
+
+GPU support
+------------
+Thanks to NVIDIA supports, training a fully connected network on a GPU, which may be 10 to 20 times faster than training them on a CPU. For convolutional network, may have 50 times faster. This requires an NVIDIA GPU with CUDA and cuDNN support.
+
+1. Installing Microsoft Visual Studio
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+You should preinstall Microsoft Visual Studio (VS) before installing CUDA. The lowest version requirements is VS2010. We recommend installing VS2015 or VS2013. CUDA7.5 supports VS2010, VS2012 and VS2013. CUDA8.0 also supports VS2015.
+
+2. Installing CUDA
+^^^^^^^^^^^^^^^^^^^^^^^
+Download and install the latest CUDA is available from NVIDIA website:
+
+`CUDA download `_
+
+We do not recommend modifying the default installation directory.
+
+3. Installing cuDNN
+^^^^^^^^^^^^^^^^^^^^^^
+The NVIDIA CUDA® Deep Neural Network library (cuDNN) is a GPU-accelerated library of primitives for deep neural networks. Download and extract the latest cuDNN is available from NVIDIA website:
+
+`cuDNN download `_
+
+After extracting cuDNN, you will get three folders (bin, lib, include). Then these folders should be copied to CUDA installation. (The default installation directory is `C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v8.0`)
+
+Installing TensorLayer
+------------------------
+For TensorLayer, please refer to the steps mentioned above.
+
+.. code-block:: bash
+
+ pip install tensorflow #CPU version
+ pip install tensorflow-gpu #GPU version (GPU version and CPU version just choose one)
+ pip install tensorlayer #Install tensorlayer
+
+
+
+Issue
+=======
+
+If you get the following output when import tensorlayer, please read `FQA `_.
+
+.. code-block:: bash
+
+ _tkinter.TclError: no display name and no $DISPLAY environment variable
diff --git a/docs/user/my_figs/basic_seq2seq.png b/docs/user/my_figs/basic_seq2seq.png
new file mode 100644
index 0000000..4e59bcf
Binary files /dev/null and b/docs/user/my_figs/basic_seq2seq.png differ
diff --git a/docs/user/my_figs/img_tensorlayer.png b/docs/user/my_figs/img_tensorlayer.png
new file mode 100644
index 0000000..c6b1f7c
Binary files /dev/null and b/docs/user/my_figs/img_tensorlayer.png differ
diff --git a/docs/user/my_figs/img_tlayer.png b/docs/user/my_figs/img_tlayer.png
new file mode 100644
index 0000000..7b998ea
Binary files /dev/null and b/docs/user/my_figs/img_tlayer.png differ
diff --git a/docs/user/my_figs/img_tlayer_big.png b/docs/user/my_figs/img_tlayer_big.png
new file mode 100644
index 0000000..3679b70
Binary files /dev/null and b/docs/user/my_figs/img_tlayer_big.png differ
diff --git a/docs/user/my_figs/img_tunelayer.png b/docs/user/my_figs/img_tunelayer.png
new file mode 100644
index 0000000..7b998ea
Binary files /dev/null and b/docs/user/my_figs/img_tunelayer.png differ
diff --git a/docs/user/my_figs/karpathy_rnn.jpeg b/docs/user/my_figs/karpathy_rnn.jpeg
new file mode 100644
index 0000000..479a125
Binary files /dev/null and b/docs/user/my_figs/karpathy_rnn.jpeg differ
diff --git a/docs/user/my_figs/mnist.jpeg b/docs/user/my_figs/mnist.jpeg
new file mode 100644
index 0000000..0803233
Binary files /dev/null and b/docs/user/my_figs/mnist.jpeg differ
diff --git a/docs/user/my_figs/pong_game.jpeg b/docs/user/my_figs/pong_game.jpeg
new file mode 100644
index 0000000..c6ee4b3
Binary files /dev/null and b/docs/user/my_figs/pong_game.jpeg differ
diff --git a/docs/user/my_figs/seq2seq.png b/docs/user/my_figs/seq2seq.png
new file mode 100644
index 0000000..41e902c
Binary files /dev/null and b/docs/user/my_figs/seq2seq.png differ
diff --git a/docs/user/my_figs/tl_black_logo.png b/docs/user/my_figs/tl_black_logo.png
new file mode 100644
index 0000000..8bf892d
Binary files /dev/null and b/docs/user/my_figs/tl_black_logo.png differ
diff --git a/docs/user/my_figs/tl_transparent_logo.png b/docs/user/my_figs/tl_transparent_logo.png
new file mode 100644
index 0000000..359a232
Binary files /dev/null and b/docs/user/my_figs/tl_transparent_logo.png differ
diff --git a/docs/user/my_figs/tl_white_logo.png b/docs/user/my_figs/tl_white_logo.png
new file mode 100644
index 0000000..a42806d
Binary files /dev/null and b/docs/user/my_figs/tl_white_logo.png differ
diff --git a/docs/user/my_figs/tsne.png b/docs/user/my_figs/tsne.png
new file mode 100644
index 0000000..848d7af
Binary files /dev/null and b/docs/user/my_figs/tsne.png differ
diff --git a/docs/user/my_figs/word2vec_basic.pdf b/docs/user/my_figs/word2vec_basic.pdf
new file mode 100644
index 0000000..6dc5b92
Binary files /dev/null and b/docs/user/my_figs/word2vec_basic.pdf differ
diff --git a/examples/basic_tutorials/tutorial_LayerList.py b/examples/basic_tutorials/tutorial_LayerList.py
new file mode 100644
index 0000000..2b60fec
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_LayerList.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from tensorlayer.layers import LayerList
+from tensorlayer.layers import Dense
+import tensorlayer as tl
+import numpy as np
+
+layer_list = []
+layer_list.append(Dense(n_units=800, act=tl.ReLU, in_channels=784, name='Dense1'))
+layer_list.append(Dense(n_units=800, act=tl.ReLU, in_channels=800, name='Dense2'))
+layer_list.append(Dense(n_units=10, act=tl.ReLU, in_channels=800, name='Dense3'))
+MLP = LayerList(layer_list)
+
+X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 784))
+
+def generator_train():
+ inputs = X_train
+ targets = y_train
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ yield (_input, np.array(_target))
+
+n_epoch = 50
+batch_size = 128
+print_freq = 2
+shuffle_buffer_size = 128
+
+# train_weights = MLP.trainable_weights
+# print(train_weights)
+optimizer = tl.optimizers.Momentum(0.05, 0.9)
+train_ds = tl.dataflow.FromGenerator(
+ generator_train, output_types=(tl.float32, tl.int32) , column_names=['data', 'label']
+)
+train_ds = tl.dataflow.Shuffle(train_ds,shuffle_buffer_size)
+train_ds = tl.dataflow.Batch(train_ds,batch_size)
+
+
+model = tl.models.Model(network=MLP, loss_fn=tl.cost.cross_entropy, optimizer=optimizer)
+model.train(n_epoch=n_epoch, train_dataset=train_ds, print_freq=print_freq, print_train_batch=False)
+model.save_weights('./model.npz', format='npz_dict')
+model.load_weights('./model.npz', format='npz_dict')
\ No newline at end of file
diff --git a/examples/basic_tutorials/tutorial_cifar10_cnn_dynamic_MS_backend.py b/examples/basic_tutorials/tutorial_cifar10_cnn_dynamic_MS_backend.py
new file mode 100644
index 0000000..02ab3e8
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_cifar10_cnn_dynamic_MS_backend.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import time
+import numpy as np
+import multiprocessing
+import tensorflow as tf
+from tensorlayer.layers import Module
+import tensorlayer as tl
+from tensorlayer.layers import (Conv2d, Dense, Flatten, MaxPool2d, BatchNorm2d)
+
+from mindspore.nn import Momentum, WithLossCell
+from mindspore import ParameterTuple
+import mindspore.nn as nn
+import mindspore as ms
+from mindspore.ops import composite as C
+import mindspore.ops.operations as P
+
+# enable debug logging
+tl.logging.set_verbosity(tl.logging.DEBUG)
+tl.logging.set_verbosity(tl.logging.DEBUG)
+
+class CNN(Module):
+ def __init__(self):
+ super(CNN, self).__init__()
+ self.conv1 = Conv2d(64, (5, 5), (2, 2), padding='SAME', b_init=None, name='conv1', in_channels=3, act=tl.ReLU, data_format='channels_first')
+ self.bn = BatchNorm2d(num_features=64, act=tl.ReLU, data_format='channels_first')
+ self.maxpool1 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool1', data_format='channels_first')
+ self.conv2 = Conv2d(128, (5, 5), (2, 2), padding='SAME', act=tl.ReLU, b_init=None, name='conv2', in_channels=64, data_format='channels_first')
+ self.maxpool2 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool2', data_format='channels_first')
+
+ self.flatten = Flatten(name='flatten')
+ self.dense1 = Dense(120, act=tl.ReLU, name='dense1relu', in_channels=4608)
+ self.dense2 = Dense(84, act=tl.ReLU, name='dense2relu', in_channels=120)
+ self.dense3 = Dense(10, act=None, name='output', in_channels=84)
+
+
+ def forward(self, x):
+ z = self.conv1(x)
+ z = self.bn(z)
+ z = self.maxpool1(z)
+ z = self.conv2(z)
+ z = self.maxpool2(z)
+ z = self.flatten(z)
+ z = self.dense1(z)
+ z = self.dense2(z)
+ z = self.dense3(z)
+ return z
+
+# training settings
+batch_size = 128
+n_epoch = 500
+shuffle_buffer_size = 128
+
+
+# prepare cifar10 data
+X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3), plotable=False)
+def generator_train():
+ inputs = X_train
+ targets = y_train
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ yield _input, _target
+
+
+def generator_test():
+ inputs = X_test
+ targets = y_test
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ # yield _input.encode('utf-8'), _target.encode('utf-8')
+ yield _input, _target
+
+def _map_fn_train(img, target):
+ # 1. Randomly crop a [height, width] section of the image.
+ img = tf.image.random_crop(img, [24, 24, 3])
+ # 2. Randomly flip the image horizontally.
+ img = tf.image.random_flip_left_right(img)
+ # 3. Randomly change brightness.
+ img = tf.image.random_brightness(img, max_delta=63)
+ # 4. Randomly change contrast.
+ img = tf.image.random_contrast(img, lower=0.2, upper=1.8)
+ # 5. Subtract off the mean and divide by the variance of the pixels.
+ img = tf.image.per_image_standardization(img)
+ target = tf.reshape(target, ())
+ return img, target
+
+
+class GradWrap(Module):
+ """ GradWrap definition """
+
+ def __init__(self, network):
+ super(GradWrap, self).__init__(auto_prefix=False)
+ self.network = network
+ self.weights = ParameterTuple(filter(lambda x: x.requires_grad, network.get_parameters()))
+
+ def forward(self, x, label):
+ return C.GradOperation(get_by_list=True)(self.network, self.weights)(x, label)
+
+
+# dataset API and augmentation
+train_ds = tf.data.Dataset.from_generator(
+ generator_train, output_types=(tf.float32, tf.int32)
+) # , output_shapes=((24, 24, 3), (1)))
+train_ds = train_ds.map(_map_fn_train, num_parallel_calls=multiprocessing.cpu_count())
+# train_ds = train_ds.repeat(n_epoch)
+train_ds = train_ds.shuffle(shuffle_buffer_size)
+train_ds = train_ds.prefetch(buffer_size=4096)
+train_ds = train_ds.batch(batch_size)
+
+# get the network
+net = CNN()
+train_weights = net.trainable_weights
+# optimizer = Adam(train_weights, learning_rate=0.01)
+optimizer = Momentum(train_weights, 0.01, 0.5)
+criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
+net_with_criterion = WithLossCell(net, criterion)
+train_network = GradWrap(net_with_criterion)
+train_network.set_train()
+# print(train_weights)
+for epoch in range(n_epoch):
+ start_time = time.time()
+ train_network.set_train()
+ train_loss, train_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in train_ds:
+ X_batch = ms.Tensor(X_batch.numpy(), dtype=ms.float32)
+ y_batch = ms.Tensor(y_batch.numpy(), dtype=ms.int32)
+ X_batch = tl.nhwc_to_nchw(X_batch)
+ y_batch = tl.nhwc_to_nchw(y_batch)
+ output = net(X_batch)
+ loss_output = criterion(output, y_batch)
+ grads = train_network(X_batch, y_batch)
+ success = optimizer(grads)
+ loss = loss_output.asnumpy()
+ train_loss += loss
+ n_iter += 1
+ train_acc += np.mean((P.Equal()(P.Argmax(axis=1)(output), y_batch).asnumpy()))
+ print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
+ print(" train loss: {}".format(train_loss / n_iter))
+ print(" train acc: {}".format(train_acc / n_iter))
+ print(" loss ", loss)
+
+# start_time = time.time()
+
+# train_loss, train_acc, n_iter = 0, 0, 0
+# for X_batch, y_batch in train_ds:
+# net.set_train()
+
+# with tf.GradientTape() as tape:
+# # compute outputs
+# _logits = net(X_batch)
+# # compute loss and update model
+# _loss_ce = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
+
+# grad = tape.gradient(_loss_ce, train_weights)
+# optimizer.apply_gradients(zip(grad, train_weights))
+
+# train_loss += _loss_ce
+# train_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+# n_iter += 1
+
+# print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
+# print(" train loss: {}".format(train_loss / n_iter))
+# print(" train acc: {}".format(train_acc / n_iter))
diff --git a/examples/basic_tutorials/tutorial_cifar10_cnn_dynamic_TF_backend.py b/examples/basic_tutorials/tutorial_cifar10_cnn_dynamic_TF_backend.py
new file mode 100644
index 0000000..f399bef
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_cifar10_cnn_dynamic_TF_backend.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import time
+import numpy as np
+import multiprocessing
+import tensorflow as tf
+
+from tensorlayer.layers import Module
+import tensorlayer as tl
+from tensorlayer.layers import (Conv2d, Dense, Flatten, MaxPool2d, BatchNorm2d)
+
+# enable debug logging
+tl.logging.set_verbosity(tl.logging.DEBUG)
+tl.logging.set_verbosity(tl.logging.DEBUG)
+
+# prepare cifar10 data
+X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3), plotable=False)
+
+
+class CNN(Module):
+
+ def __init__(self):
+ super(CNN, self).__init__()
+ # weights init
+ W_init = tl.initializers.truncated_normal(stddev=5e-2)
+ W_init2 = tl.initializers.truncated_normal(stddev=0.04)
+ b_init2 = tl.initializers.constant(value=0.1)
+
+ self.conv1 = Conv2d(64, (5, 5), (1, 1), padding='SAME', W_init=W_init, b_init=None, name='conv1', in_channels=3)
+ self.bn = BatchNorm2d(num_features=64, act=tl.ReLU)
+ self.maxpool1 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool1')
+
+ self.conv2 = Conv2d(
+ 64, (5, 5), (1, 1), padding='SAME', act=tl.ReLU, W_init=W_init, b_init=None, name='conv2', in_channels=64
+ )
+ self.maxpool2 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool2')
+
+ self.flatten = Flatten(name='flatten')
+ self.dense1 = Dense(384, act=tl.ReLU, W_init=W_init2, b_init=b_init2, name='dense1relu', in_channels=2304)
+ self.dense2 = Dense(192, act=tl.ReLU, W_init=W_init2, b_init=b_init2, name='dense2relu', in_channels=384)
+ self.dense3 = Dense(10, act=None, W_init=W_init2, name='output', in_channels=192)
+
+ def forward(self, x):
+ z = self.conv1(x)
+ z = self.bn(z)
+ z = self.maxpool1(z)
+ z = self.conv2(z)
+ z = self.maxpool2(z)
+ z = self.flatten(z)
+ z = self.dense1(z)
+ z = self.dense2(z)
+ z = self.dense3(z)
+ return z
+
+
+# get the network
+net = CNN()
+
+# training settings
+batch_size = 128
+n_epoch = 500
+learning_rate = 0.0001
+print_freq = 5
+n_step_epoch = int(len(y_train) / batch_size)
+n_step = n_epoch * n_step_epoch
+shuffle_buffer_size = 128
+
+train_weights = net.trainable_weights
+optimizer = tl.optimizers.Adam(learning_rate)
+# looking for decay learning rate? see https://github.com/tensorlayer/srgan/blob/master/train.py
+
+
+def generator_train():
+ inputs = X_train
+ targets = y_train
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ # yield _input.encode('utf-8'), _target.encode('utf-8')
+ yield _input, _target
+
+
+def generator_test():
+ inputs = X_test
+ targets = y_test
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ # yield _input.encode('utf-8'), _target.encode('utf-8')
+ yield _input, _target
+
+
+def _map_fn_train(img, target):
+ # 1. Randomly crop a [height, width] section of the image.
+ img = tf.image.random_crop(img, [24, 24, 3])
+ # 2. Randomly flip the image horizontally.
+ img = tf.image.random_flip_left_right(img)
+ # 3. Randomly change brightness.
+ img = tf.image.random_brightness(img, max_delta=63)
+ # 4. Randomly change contrast.
+ img = tf.image.random_contrast(img, lower=0.2, upper=1.8)
+ # 5. Subtract off the mean and divide by the variance of the pixels.
+ img = tf.image.per_image_standardization(img)
+ target = tf.reshape(target, ())
+ return img, target
+
+
+def _map_fn_test(img, target):
+ # 1. Crop the central [height, width] of the image.
+ img = tf.image.resize_with_pad(img, 24, 24)
+ # 2. Subtract off the mean and divide by the variance of the pixels.
+ img = tf.image.per_image_standardization(img)
+ img = tf.reshape(img, (24, 24, 3))
+ target = tf.reshape(target, ())
+ return img, target
+
+
+# dataset API and augmentation
+train_ds = tf.data.Dataset.from_generator(
+ generator_train, output_types=(tf.float32, tf.int32)
+) # , output_shapes=((24, 24, 3), (1)))
+train_ds = train_ds.map(_map_fn_train, num_parallel_calls=multiprocessing.cpu_count())
+# train_ds = train_ds.repeat(n_epoch)
+train_ds = train_ds.shuffle(shuffle_buffer_size)
+train_ds = train_ds.prefetch(buffer_size=4096)
+train_ds = train_ds.batch(batch_size)
+# value = train_ds.make_one_shot_iterator().get_next()
+
+test_ds = tf.data.Dataset.from_generator(
+ generator_test, output_types=(tf.float32, tf.int32)
+) # , output_shapes=((24, 24, 3), (1)))
+# test_ds = test_ds.shuffle(shuffle_buffer_size)
+test_ds = test_ds.map(_map_fn_test, num_parallel_calls=multiprocessing.cpu_count())
+# test_ds = test_ds.repeat(n_epoch)
+test_ds = test_ds.prefetch(buffer_size=4096)
+test_ds = test_ds.batch(batch_size)
+# value_test = test_ds.make_one_shot_iterator().get_next()
+
+for epoch in range(n_epoch):
+ start_time = time.time()
+
+ train_loss, train_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in train_ds:
+ net.set_train()
+
+ with tf.GradientTape() as tape:
+ # compute outputs
+ _logits = net(X_batch)
+ # compute loss and update model
+ _loss_ce = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
+
+ grad = tape.gradient(_loss_ce, train_weights)
+ optimizer.apply_gradients(zip(grad, train_weights))
+
+ train_loss += _loss_ce
+ train_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+
+ print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
+ print(" train loss: {}".format(train_loss / n_iter))
+ print(" train acc: {}".format(train_acc / n_iter))
+
+ # use training and evaluation sets to evaluate the model every print_freq epoch
+ if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
+
+ net.eval()
+ val_loss, val_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in test_ds:
+ _logits = net(X_batch) # is_train=False, disable dropout
+ val_loss += tl.cost.cross_entropy(_logits, y_batch, name='eval_loss')
+ val_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+ print(" val loss: {}".format(val_loss / n_iter))
+ print(" val acc: {}".format(val_acc / n_iter))
+
+# use testing data to evaluate the model
+net.eval()
+test_loss, test_acc, n_iter = 0, 0, 0
+for X_batch, y_batch in test_ds:
+ _logits = net(X_batch)
+ test_loss += tl.cost.cross_entropy(_logits, y_batch, name='test_loss')
+ test_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+print(" test loss: {}".format(test_loss / n_iter))
+print(" test acc: {}".format(test_acc / n_iter))
diff --git a/examples/basic_tutorials/tutorial_mnist_mlp_dynamci_dragon.py b/examples/basic_tutorials/tutorial_mnist_mlp_dynamci_dragon.py
new file mode 100644
index 0000000..9c06ec5
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_mnist_mlp_dynamci_dragon.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+os.environ['TL_BACKEND'] = 'dragon'
+
+from tensorlayer.layers import Module
+from tensorlayer.layers import Dense
+import tensorlayer as tl
+import dragon as dg
+import time
+import argparse
+import numpy as np
+
+X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 784))
+
+
+class CustomModel(Module):
+
+ def __init__(self):
+ super(CustomModel, self).__init__()
+ self.dense1 = Dense(n_units=800, act=tl.ReLU, in_channels=784)
+ self.dense2 = Dense(n_units=800, act=tl.ReLU, in_channels=800)
+ self.dense3 = Dense(n_units=10, act=tl.ReLU, in_channels=800)
+
+ def forward(self, x, foo=None):
+ z = self.dense1(x)
+ z = self.dense2(z)
+ out = self.dense3(z)
+ return out
+
+
+def parse_args():
+ """Parse the arguments."""
+ parser = argparse.ArgumentParser(description='Train a cifar10 resnet')
+ parser.add_argument('--execution', default='EAGER_MODE', type=str, help='The execution mode')
+ parser.add_argument('--seed', default=1337, type=int, help='The random seed')
+ parser.add_argument('--cuda', default=-1, type=int, help='The cuda device to use')
+ return parser.parse_args()
+
+
+class Classifier(object):
+ """The base classifier class."""
+
+ # TensorSpec for graph execution
+ image_spec = dg.Tensor([None, 3, 32, 32], 'float32')
+ label_spec = dg.Tensor([None], 'int64')
+
+ def __init__(self, optimizer):
+ super(Classifier, self).__init__()
+ self.net = CustomModel()
+ self.optimizer = optimizer
+ self.params = self.net.trainable_weights
+
+ def step(self, image, label):
+ with dg.GradientTape() as tape:
+ logit = self.net(image)
+ # logit = dg.cast(logit, 'float64')
+ logit = dg.cast(dg.math.argmax(logit, -1), 'int64')
+ label = dg.cast(label, 'int64')
+ # print("logit :\n", logit, label)
+ # loss = dg.losses.smooth_l1_loss([logit, label])
+ loss = dg.math.sum(logit - label) # dg.losses.sparse_softmax_cross_entropy([logit, label])
+ accuracy = dg.math.mean(dg.math.equal([logit, label]).astype('float32'))
+ grads = tape.gradient(loss, self.params)
+ self.optimizer.apply_gradients(zip(self.params, grads))
+ return loss, accuracy, self.optimizer
+
+
+if __name__ == '__main__':
+ args = parse_args()
+ dg.logging.info('Called with args:\n' + str(args))
+
+ np.random.seed(args.seed)
+ dg.autograph.set_execution(args.execution)
+ dg.cuda.set_default_device(args.cuda)
+
+ # Define the model
+ model = Classifier(dg.optimizers.SGD(base_lr=0.01, momentum=0.9, weight_decay=1e-4))
+
+ # Compile for graph execution if necessary
+ if args.execution == 'GRAPH_MODE':
+ model.step = dg.function(
+ func=model.step,
+ input_signature=[model.image_spec, model.label_spec],
+ )
+
+ # Main loop
+ import tensorflow as tf
+ batch_size = 200
+ for i in range(50):
+ for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
+ image = dg.EagerTensor(X_batch, copy=False)
+ label = dg.EagerTensor(y_batch, copy=False, dtype='float32')
+ loss, accuracy, _ = model.step(image, label)
+ if i % 20 == 0:
+ dg.logging.info(
+ 'Iteration %d, lr = %s, loss = %.5f, accuracy = %.3f' %
+ (i, str(model.optimizer.base_lr), loss, accuracy)
+ )
diff --git a/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_MS_backend.py b/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_MS_backend.py
new file mode 100644
index 0000000..e480221
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_MS_backend.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import mindspore.nn as nn
+import mindspore.ops.operations as P
+from mindspore.ops import composite as C
+from mindspore.common import dtype as mstype
+from mindspore import context, Tensor, ParameterTuple
+from mindspore.common.initializer import TruncatedNormal
+from mindspore.nn import SoftmaxCrossEntropyWithLogits, Momentum, WithLossCell
+
+import numpy as np
+import tensorlayer as tl
+import mindspore as ms
+import tensorflow as tf
+import time
+from tensorlayer.layers import Module
+from tensorlayer.layers import Dense
+import mindspore.nn as nn
+
+
+class MLP(Module):
+
+ def __init__(self):
+ super(MLP, self).__init__()
+ self.dense1 = Dense(n_units=800, act=tl.ReLU, in_channels=784)
+ self.dense2 = Dense(n_units=800, act=tl.ReLU, in_channels=800)
+ self.dense3 = Dense(n_units=10, act=tl.ReLU, in_channels=800)
+
+ def forward(self, x):
+ z = self.dense1(x)
+ z = self.dense2(z)
+ out = self.dense3(z)
+ return out
+
+
+class GradWrap(Module):
+ """ GradWrap definition """
+
+ def __init__(self, network):
+ super(GradWrap, self).__init__(auto_prefix=False)
+ self.network = network
+ self.weights = ParameterTuple(filter(lambda x: x.requires_grad, network.get_parameters()))
+
+ def forward(self, x, label):
+ return C.GradOperation(get_by_list=True)(self.network, self.weights)(x, label)
+
+
+def generator_train():
+ inputs = X_train
+ targets = y_train
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ yield _input, _target
+
+
+net = MLP()
+train_weights = list(filter(lambda x: x.requires_grad, net.get_parameters()))
+optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.15, 0.8)
+
+criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
+net_with_criterion = WithLossCell(net, criterion)
+train_network = GradWrap(net_with_criterion)
+train_network.set_train()
+
+X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 784))
+train_ds = tf.data.Dataset.from_generator(generator_train, output_types=(tf.float32, tf.int32))
+shuffle_buffer_size = 128
+batch_size = 128
+train_ds = train_ds.shuffle(shuffle_buffer_size)
+train_ds = train_ds.batch(batch_size)
+n_epoch = 50
+
+for epoch in range(n_epoch):
+ start_time = time.time()
+ train_network.set_train()
+ train_loss, train_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in train_ds:
+ X_batch = ms.Tensor(X_batch.numpy(), dtype=ms.float32)
+ y_batch = ms.Tensor(y_batch.numpy(), dtype=ms.int32)
+ output = net(X_batch)
+ loss_output = criterion(output, y_batch)
+ grads = train_network(X_batch, y_batch)
+ success = optimizer(grads)
+ loss = loss_output.asnumpy()
+ train_loss += loss
+ n_iter += 1
+ train_acc += np.mean((P.Equal()(P.Argmax(axis=1)(output), y_batch).asnumpy()))
+ print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
+ print(" train loss: {}".format(train_loss / n_iter))
+ print(" train acc: {}".format(train_acc / n_iter))
+ print(" loss ", loss)
diff --git a/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_TF_backend.py b/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_TF_backend.py
new file mode 100644
index 0000000..1287391
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_TF_backend.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import time
+
+import tensorflow as tf
+import tensorlayer as tl
+from tensorlayer.layers import Module
+from tensorlayer.layers import Dense, Dropout, BatchNorm1d
+
+X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 784))
+
+
+class CustomModel(Module):
+
+ def __init__(self):
+ super(CustomModel, self).__init__()
+ self.dropout1 = Dropout(keep=0.8)
+ self.dense1 = Dense(n_units=800, in_channels=784)
+ self.batchnorm = BatchNorm1d(act=tl.ReLU, num_features=800)
+ self.dropout2 = Dropout(keep=0.8)
+ self.dense2 = Dense(n_units=800, act=tl.ReLU, in_channels=800)
+ self.dropout3 = Dropout(keep=0.8)
+ self.dense3 = Dense(n_units=10, act=tl.ReLU, in_channels=800)
+
+ def forward(self, x, foo=None):
+ z = self.dropout1(x)
+ z = self.dense1(z)
+ z = self.batchnorm(z)
+ z = self.dropout2(z)
+ z = self.dense2(z)
+ z = self.dropout3(z)
+ out = self.dense3(z)
+ if foo is not None:
+ out = tl.ops.relu(out)
+ return out
+
+
+MLP = CustomModel()
+n_epoch = 50
+batch_size = 500
+print_freq = 5
+train_weights = MLP.trainable_weights
+optimizer = tl.optimizers.Adam(lr=0.0001)
+
+for epoch in range(n_epoch): ## iterate the dataset n_epoch times
+ start_time = time.time()
+ ## iterate over the entire training set once (shuffle the data via training)
+ for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
+ MLP.set_train() # enable dropout
+ with tf.GradientTape() as tape:
+ ## compute outputs
+ _logits = MLP(X_batch)
+ ## compute loss and update model
+ _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
+ grad = tape.gradient(_loss, train_weights)
+ optimizer.apply_gradients(zip(grad, train_weights))
+
+ ## use training and evaluation sets to evaluate the model every print_freq epoch
+ if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
+ MLP.set_train()
+ print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
+ train_loss, train_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
+ _logits = MLP(X_batch)
+ train_loss += tl.cost.cross_entropy(_logits, y_batch, name='eval_loss')
+ train_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+ print(" train loss: {}".format(train_loss / n_iter))
+ print(" train acc: {}".format(train_acc / n_iter))
+
+ val_loss, val_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=False):
+ _logits = MLP(X_batch) # is_train=False, disable dropout
+ val_loss += tl.cost.cross_entropy(_logits, y_batch, name='eval_loss')
+ val_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+ print(" val loss: {}".format(val_loss / n_iter))
+ print(" val acc: {}".format(val_acc / n_iter))
+
+## use testing data to evaluate the model
+MLP.eval()
+test_loss, test_acc, n_iter = 0, 0, 0
+for X_batch, y_batch in tl.iterate.minibatches(X_test, y_test, batch_size, shuffle=False):
+ _logits = MLP(X_batch, foo=1)
+ test_loss += tl.cost.cross_entropy(_logits, y_batch, name='test_loss')
+ test_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+print(" test foo=1 loss: {}".format(val_loss / n_iter))
+print(" test foo=1 acc: {}".format(val_acc / n_iter))
diff --git a/examples/basic_tutorials/tutorial_mnist_mlp_mindspore.py b/examples/basic_tutorials/tutorial_mnist_mlp_mindspore.py
new file mode 100644
index 0000000..3e552d3
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_mnist_mlp_mindspore.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import mindspore.nn as nn
+import mindspore.ops.operations as P
+from mindspore.ops import composite as C
+from mindspore.common import dtype as mstype
+from mindspore import context, Tensor, ParameterTuple
+from mindspore.common.initializer import TruncatedNormal
+from mindspore.nn import Dense, WithLossCell, SoftmaxCrossEntropyWithLogits, Momentum
+import tensorlayer as tl
+import mindspore as ms
+import tensorflow as tf
+import time
+
+context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU")
+
+
+def fc_with_initialize(input_channels, out_channels):
+ """weight initial for fc layer"""
+ weight = weight_variable()
+ bias = weight_variable()
+ return nn.Dense(input_channels, out_channels, weight, bias)
+
+
+def weight_variable():
+ """weight initial"""
+ return TruncatedNormal(0.02)
+
+
+class MLP(nn.Cell):
+ """
+ Lenet network
+ Args:
+ num_class (int): Num classes. Default: 10.
+
+ Returns:
+ Tensor, output tensor
+
+ Examples:
+ >>> MLP(num_class=10)
+ """
+
+ def __init__(self, num_class=10):
+ super(MLP, self).__init__()
+ self.num_class = num_class
+ self.fc1 = fc_with_initialize(784, 800)
+ self.fc2 = fc_with_initialize(800, 800)
+ self.fc3 = fc_with_initialize(800, self.num_class)
+ self.relu = nn.ReLU()
+
+ def construct(self, x):
+ x = self.fc1(x)
+ x = self.relu(x)
+ x = self.fc2(x)
+ x = self.relu(x)
+ x = self.fc3(x)
+ return x
+
+
+class GradWrap(nn.Cell):
+ """ GradWrap definition """
+
+ def __init__(self, network):
+ super(GradWrap, self).__init__(auto_prefix=False)
+ self.network = network
+ self.weights = ParameterTuple(filter(lambda x: x.requires_grad, network.get_parameters()))
+
+ def construct(self, x, label):
+ weights = self.weights
+ return C.GradOperation('get_by_list', get_by_list=True)(self.network, weights)(x, label)
+
+
+def generator_train():
+ inputs = X_train
+ targets = y_train
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ yield _input, _target
+
+
+net = MLP()
+optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.1, 0.9)
+criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
+net_with_criterion = WithLossCell(net, criterion)
+train_network = GradWrap(net_with_criterion)
+train_network.set_train()
+
+X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 784))
+train_ds = tf.data.Dataset.from_generator(generator_train, output_types=(tf.float32, tf.int32))
+shuffle_buffer_size = 128
+batch_size = 128
+train_ds = train_ds.shuffle(shuffle_buffer_size)
+train_ds = train_ds.batch(batch_size)
+n_epoch = 50
+
+for epoch in range(n_epoch):
+ start_time = time.time()
+ train_network.set_train()
+ train_loss, train_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in train_ds:
+ X_batch = ms.Tensor(X_batch.numpy(), dtype=ms.float32)
+ y_batch = ms.Tensor(y_batch.numpy(), dtype=ms.int32)
+ output = net(X_batch)
+ loss_output = criterion(output, y_batch)
+ grads = train_network(X_batch, y_batch)
+ success = optimizer(grads)
+ loss = loss_output.asnumpy()
+ train_loss += loss
+ n_iter += 1
+ # train_acc += np.mean((P.Equal()(P.Argmax(axis=1)(output), y_batch).asnumpy()))
+ print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
+ print(" train loss: {}".format(train_loss / n_iter))
+ # print(" train acc: {}".format(train_acc / n_iter))
+ print(" triain weights ", train_network.trainable_params()[0].data)
diff --git a/examples/basic_tutorials/tutorial_mnist_simple.py b/examples/basic_tutorials/tutorial_mnist_simple.py
new file mode 100644
index 0000000..4d2bc7c
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_mnist_simple.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import os
+os.environ['TL_BACKEND'] = 'tensorflow'
+# os.environ['TL_BACKEND'] = 'mindspore'
+
+import tensorlayer as tl
+from tensorlayer.layers import Module
+from tensorlayer.layers import Dense, Dropout
+
+X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 784))
+
+
+class CustomModel(Module):
+
+ def __init__(self):
+ super(CustomModel, self).__init__()
+ self.dropout1 = Dropout(keep=0.8)
+ self.dense1 = Dense(n_units=800, act=tl.ReLU, in_channels=784)
+ self.dropout2 = Dropout(keep=0.8)
+ self.dense2 = Dense(n_units=800, act=tl.ReLU, in_channels=800)
+ self.dropout3 = Dropout(keep=0.8)
+ self.dense3 = Dense(n_units=10, act=tl.ReLU, in_channels=800)
+
+ def forward(self, x, foo=None):
+ z = self.dropout1(x)
+ z = self.dense1(z)
+ # z = self.bn(z)
+ z = self.dropout2(z)
+ z = self.dense2(z)
+ z = self.dropout3(z)
+ out = self.dense3(z)
+ if foo is not None:
+ out = tl.ops.relu(out)
+ return out
+
+
+def generator_train():
+ inputs = X_train
+ targets = y_train
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ yield (_input, np.array(_target))
+
+
+MLP = CustomModel()
+
+n_epoch = 50
+batch_size = 128
+print_freq = 2
+shuffle_buffer_size = 128
+
+train_weights = MLP.trainable_weights
+optimizer = tl.optimizers.Momentum(0.05, 0.9)
+train_ds = tl.dataflow.FromGenerator(
+ generator_train, output_types=(tl.float32, tl.int32) , column_names=['data', 'label']
+)
+train_ds = tl.dataflow.Shuffle(train_ds,shuffle_buffer_size)
+train_ds = tl.dataflow.Batch(train_ds,batch_size)
+
+
+model = tl.models.Model(network=MLP, loss_fn=tl.cost.cross_entropy, optimizer=optimizer)
+model.train(n_epoch=n_epoch, train_dataset=train_ds, print_freq=print_freq, print_train_batch=False)
+model.save_weights('./model.npz', format='npz_dict')
+model.load_weights('./model.npz', format='npz_dict')
diff --git a/examples/basic_tutorials/tutorial_nested_usage_of_Layer.py b/examples/basic_tutorials/tutorial_nested_usage_of_Layer.py
new file mode 100644
index 0000000..27ae9be
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_nested_usage_of_Layer.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import time
+import numpy as np
+import multiprocessing
+import tensorflow as tf
+
+from tensorlayer.layers import Module, SequentialLayer
+import tensorlayer as tl
+from tensorlayer.layers import (Conv2d, Dense, Flatten, MaxPool2d, BatchNorm2d, Elementwise)
+
+X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3), plotable=False)
+
+class Block(Module):
+ def __init__(self, in_channels):
+ super(Block, self).__init__()
+ self.dense1 = Dense(in_channels=in_channels, n_units=256)
+ self.dense2 = Dense(in_channels=256, n_units=384)
+ self.dense3 = Dense(in_channels=in_channels, n_units=384)
+ self.concat = Elementwise(combine_fn=tl.ops.add)
+
+ def forward(self, inputs):
+ z = self.dense1(inputs)
+ z1 = self.dense2(z)
+
+ z2 = self.dense3(inputs)
+ out = self.concat([z1, z2])
+ return out
+
+
+class CNN(Module):
+
+ def __init__(self):
+ super(CNN, self).__init__()
+ # weights init
+ W_init = tl.initializers.truncated_normal(stddev=5e-2)
+ W_init2 = tl.initializers.truncated_normal(stddev=0.04)
+ b_init2 = tl.initializers.constant(value=0.1)
+
+ self.conv1 = Conv2d(64, (5, 5), (1, 1), padding='SAME', W_init=W_init, b_init=None, name='conv1', in_channels=3)
+ self.bn = BatchNorm2d(num_features=64, act=tl.ReLU)
+ self.maxpool1 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool1')
+
+ self.conv2 = Conv2d(
+ 64, (5, 5), (1, 1), padding='SAME', act=tl.ReLU, W_init=W_init, b_init=None, name='conv2', in_channels=64
+ )
+ self.maxpool2 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool2')
+
+ self.flatten = Flatten(name='flatten')
+ self.dense1 = Dense(384, act=tl.ReLU, W_init=W_init2, b_init=b_init2, name='dense1relu', in_channels=2304)
+ self.dense_add = self.make_layer(in_channel=384)
+
+ self.dense2 = Dense(192, act=tl.ReLU, W_init=W_init2, b_init=b_init2, name='dense2relu', in_channels=384)
+ self.dense3 = Dense(10, act=None, W_init=W_init2, name='output', in_channels=192)
+
+ def forward(self, x):
+ z = self.conv1(x)
+ z = self.bn(z)
+ z = self.maxpool1(z)
+ z = self.conv2(z)
+ z = self.maxpool2(z)
+ z = self.flatten(z)
+ z = self.dense1(z)
+ z = self.dense_add(z)
+
+ z = self.dense2(z)
+ z = self.dense3(z)
+ return z
+
+ def make_layer(self, in_channel):
+ layers = []
+
+ _block = Block(in_channel)
+ layers.append(_block)
+
+ for _ in range(1, 3):
+ range_block = Block(in_channel)
+ layers.append(range_block)
+
+ return SequentialLayer(layers)
+
+
+# get the network
+net = CNN()
+# training settings
+batch_size = 128
+n_epoch = 500
+learning_rate = 0.0001
+print_freq = 5
+n_step_epoch = int(len(y_train) / batch_size)
+n_step = n_epoch * n_step_epoch
+shuffle_buffer_size = 128
+
+train_weights = net.trainable_weights
+optimizer = tl.optimizers.Adam(learning_rate)
+
+
+def generator_train():
+ inputs = X_train
+ targets = y_train
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ # yield _input.encode('utf-8'), _target.encode('utf-8')
+ yield _input, _target
+
+
+def generator_test():
+ inputs = X_test
+ targets = y_test
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ # yield _input.encode('utf-8'), _target.encode('utf-8')
+ yield _input, _target
+
+
+def _map_fn_train(img, target):
+ # 1. Randomly crop a [height, width] section of the image.
+ img = tf.image.random_crop(img, [24, 24, 3])
+ # 2. Randomly flip the image horizontally.
+ img = tf.image.random_flip_left_right(img)
+ # 3. Randomly change brightness.
+ img = tf.image.random_brightness(img, max_delta=63)
+ # 4. Randomly change contrast.
+ img = tf.image.random_contrast(img, lower=0.2, upper=1.8)
+ # 5. Subtract off the mean and divide by the variance of the pixels.
+ img = tf.image.per_image_standardization(img)
+ target = tf.reshape(target, ())
+ return img, target
+
+
+def _map_fn_test(img, target):
+ # 1. Crop the central [height, width] of the image.
+ img = tf.image.resize_with_pad(img, 24, 24)
+ # 2. Subtract off the mean and divide by the variance of the pixels.
+ img = tf.image.per_image_standardization(img)
+ img = tf.reshape(img, (24, 24, 3))
+ target = tf.reshape(target, ())
+ return img, target
+
+
+# dataset API and augmentation
+train_ds = tf.data.Dataset.from_generator(
+ generator_train, output_types=(tf.float32, tf.int32)
+) # , output_shapes=((24, 24, 3), (1)))
+train_ds = train_ds.map(_map_fn_train, num_parallel_calls=multiprocessing.cpu_count())
+# train_ds = train_ds.repeat(n_epoch)
+train_ds = train_ds.shuffle(shuffle_buffer_size)
+train_ds = train_ds.prefetch(buffer_size=4096)
+train_ds = train_ds.batch(batch_size)
+# value = train_ds.make_one_shot_iterator().get_next()
+
+test_ds = tf.data.Dataset.from_generator(
+ generator_test, output_types=(tf.float32, tf.int32)
+) # , output_shapes=((24, 24, 3), (1)))
+# test_ds = test_ds.shuffle(shuffle_buffer_size)
+test_ds = test_ds.map(_map_fn_test, num_parallel_calls=multiprocessing.cpu_count())
+# test_ds = test_ds.repeat(n_epoch)
+test_ds = test_ds.prefetch(buffer_size=4096)
+test_ds = test_ds.batch(batch_size)
+# value_test = test_ds.make_one_shot_iterator().get_next()
+
+for epoch in range(n_epoch):
+ start_time = time.time()
+
+ train_loss, train_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in train_ds:
+ net.set_train()
+
+ with tf.GradientTape() as tape:
+ # compute outputs
+ _logits = net(X_batch)
+ # compute loss and update model
+ _loss_ce = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
+
+ grad = tape.gradient(_loss_ce, train_weights)
+ optimizer.apply_gradients(zip(grad, train_weights))
+
+ train_loss += _loss_ce
+ train_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+
+ print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
+ print(" train loss: {}".format(train_loss / n_iter))
+ print(" train acc: {}".format(train_acc / n_iter))
+
+ # use training and evaluation sets to evaluate the model every print_freq epoch
+ if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
+
+ net.eval()
+ val_loss, val_acc, n_iter = 0, 0, 0
+ for X_batch, y_batch in test_ds:
+ _logits = net(X_batch) # is_train=False, disable dropout
+ val_loss += tl.cost.cross_entropy(_logits, y_batch, name='eval_loss')
+ val_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+ print(" val loss: {}".format(val_loss / n_iter))
+ print(" val acc: {}".format(val_acc / n_iter))
+
+# use testing data to evaluate the model
+net.eval()
+test_loss, test_acc, n_iter = 0, 0, 0
+for X_batch, y_batch in test_ds:
+ _logits = net(X_batch)
+ test_loss += tl.cost.cross_entropy(_logits, y_batch, name='test_loss')
+ test_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
+ n_iter += 1
+print(" test loss: {}".format(test_loss / n_iter))
+print(" test acc: {}".format(test_acc / n_iter))
\ No newline at end of file
diff --git a/examples/basic_tutorials/tutorial_paddle_tensorlayer_mlp.py b/examples/basic_tutorials/tutorial_paddle_tensorlayer_mlp.py
new file mode 100644
index 0000000..96d57b5
--- /dev/null
+++ b/examples/basic_tutorials/tutorial_paddle_tensorlayer_mlp.py
@@ -0,0 +1,57 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+import os
+os.environ['TL_BACKEND'] = 'paddle'
+
+import paddle.nn.functional as F
+from paddle.vision.transforms import Compose, Normalize
+import paddle
+
+import tensorlayer as tl
+from tensorlayer.layers import Module
+from tensorlayer.layers import Dense, Flatten
+
+transform = Compose([Normalize(mean=[127.5],
+ std=[127.5],
+ data_format='CHW')])
+print('download training data and load training data')
+train_dataset = paddle.vision.datasets.MNIST(mode='train', transform=transform)
+test_dataset = paddle.vision.datasets.MNIST(mode='test', transform=transform)
+print('load finished')
+
+class MLP(Module):
+ def __init__(self):
+ super(MLP, self).__init__()
+ self.linear1 = Dense(n_units=120, in_channels=784, act=tl.ReLU)
+ self.linear2 = Dense(n_units=84, in_channels=120, act=tl.ReLU)
+ self.linear3 = Dense(n_units=10, in_channels=84)
+ self.flatten = Flatten()
+
+ def forward(self, x):
+ x = self.flatten(x)
+ x = self.linear1(x)
+ x = self.linear2(x)
+ x = self.linear3(x)
+ return x
+
+train_loader = paddle.io.DataLoader(train_dataset, batch_size=64, shuffle=True)
+
+def train(model):
+ model.train()
+ epochs = 2
+ optim = paddle.optimizer.Adam(learning_rate=0.001, parameters=model.trainable_weights)
+ for epoch in range(epochs):
+ for batch_id, data in enumerate(train_loader()):
+ x_data = data[0]
+ y_data = data[1]
+ predicts = model(x_data)
+ loss = tl.cost.mean_squared_error(predicts, y_data)
+ acc = paddle.metric.accuracy(predicts, y_data)
+ loss.backward()
+ if batch_id % 5 == 0:
+ print("epoch: {}, batch_id: {}, loss is: {}, acc is: {}".format(epoch, batch_id, loss.numpy(), acc.numpy()))
+ optim.step()
+ optim.clear_grad()
+model = MLP()
+train(model)
+
diff --git a/examples/model_zoo/__init__.py b/examples/model_zoo/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/examples/model_zoo/common.py b/examples/model_zoo/common.py
new file mode 100644
index 0000000..7bc1bfd
--- /dev/null
+++ b/examples/model_zoo/common.py
@@ -0,0 +1,287 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import tensorflow as tf
+import colorsys, random, cv2
+import numpy as np
+from tensorlayer.visualize import save_image
+
+def decode_tf(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=0, XYSCALE=[1, 1, 1]):
+ batch_size = tf.shape(conv_output)[0]
+ conv_output = tf.reshape(conv_output, (batch_size, output_size, output_size, 3, 5 + NUM_CLASS))
+
+ conv_raw_dxdy, conv_raw_dwdh, conv_raw_conf, conv_raw_prob = tf.split(conv_output, (2, 2, 1, NUM_CLASS), axis=-1)
+
+ xy_grid = tf.meshgrid(tf.range(output_size), tf.range(output_size))
+ xy_grid = tf.expand_dims(tf.stack(xy_grid, axis=-1), axis=2) # [gx, gy, 1, 2]
+ xy_grid = tf.tile(tf.expand_dims(xy_grid, axis=0), [batch_size, 1, 1, 3, 1])
+
+ xy_grid = tf.cast(xy_grid, tf.float32)
+
+ pred_xy = ((tf.sigmoid(conv_raw_dxdy) * XYSCALE[i]) - 0.5 * (XYSCALE[i] - 1) + xy_grid) * \
+ STRIDES[i]
+ pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i])
+ pred_xywh = tf.concat([pred_xy, pred_wh], axis=-1)
+
+ pred_conf = tf.sigmoid(conv_raw_conf)
+ pred_prob = tf.sigmoid(conv_raw_prob)
+
+ pred_prob = pred_conf * pred_prob
+ pred_prob = tf.reshape(pred_prob, (batch_size, -1, NUM_CLASS))
+ pred_xywh = tf.reshape(pred_xywh, (batch_size, -1, 4))
+
+ return pred_xywh, pred_prob
+
+
+def decode(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE=[1, 1, 1]):
+ return decode_tf(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=i, XYSCALE=XYSCALE)
+
+
+def filter_boxes(box_xywh, scores, score_threshold=0.4, input_shape=tf.constant([416, 416])):
+ scores_max = tf.math.reduce_max(scores, axis=-1)
+
+ mask = scores_max >= score_threshold
+ class_boxes = tf.boolean_mask(box_xywh, mask)
+ pred_conf = tf.boolean_mask(scores, mask)
+ class_boxes = tf.reshape(class_boxes, [tf.shape(scores)[0], -1, tf.shape(class_boxes)[-1]])
+ pred_conf = tf.reshape(pred_conf, [tf.shape(scores)[0], -1, tf.shape(pred_conf)[-1]])
+
+ box_xy, box_wh = tf.split(class_boxes, (2, 2), axis=-1)
+
+ input_shape = tf.cast(input_shape, dtype=tf.float32)
+ box_yx = box_xy[..., ::-1]
+ box_hw = box_wh[..., ::-1]
+
+ box_mins = (box_yx - (box_hw / 2.)) / input_shape
+ box_maxes = (box_yx + (box_hw / 2.)) / input_shape
+ boxes = tf.concat(
+ [
+ box_mins[..., 0:1], # y_min
+ box_mins[..., 1:2], # x_min
+ box_maxes[..., 0:1], # y_max
+ box_maxes[..., 1:2] # x_max
+ ],
+ axis=-1
+ )
+ # return tf.concat([boxes, pred_conf], axis=-1)
+ return (boxes, pred_conf)
+
+
+def read_class_names(class_file_name):
+ names = {}
+ with open(class_file_name, 'r') as data:
+ for ID, name in enumerate(data):
+ names[ID] = name.strip('\n')
+ return names
+
+
+def draw_bbox(image, bboxes, show_label=True):
+ classes = read_class_names('model/coco.names')
+ num_classes = len(classes)
+ image_h, image_w, _ = image.shape
+ hsv_tuples = [(1.0 * x / num_classes, 1., 1.) for x in range(num_classes)]
+ colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
+ colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors))
+
+ random.seed(0)
+ random.shuffle(colors)
+ random.seed(None)
+
+ out_boxes, out_scores, out_classes, num_boxes = bboxes
+ for i in range(num_boxes[0]):
+ if int(out_classes[0][i]) < 0 or int(out_classes[0][i]) > num_classes: continue
+ coor = out_boxes[0][i]
+ coor[0] = int(coor[0] * image_h)
+ coor[2] = int(coor[2] * image_h)
+ coor[1] = int(coor[1] * image_w)
+ coor[3] = int(coor[3] * image_w)
+
+ fontScale = 0.5
+ score = out_scores[0][i]
+ class_ind = int(out_classes[0][i])
+ bbox_color = colors[class_ind]
+ bbox_thick = int(0.6 * (image_h + image_w) / 600)
+ c1, c2 = (coor[1], coor[0]), (coor[3], coor[2])
+ cv2.rectangle(image, c1, c2, bbox_color, bbox_thick)
+
+ if show_label:
+ bbox_mess = '%s: %.2f' % (classes[class_ind], score)
+ t_size = cv2.getTextSize(bbox_mess, 0, fontScale, thickness=bbox_thick // 2)[0]
+ c3 = (c1[0] + t_size[0], c1[1] - t_size[1] - 3)
+ cv2.rectangle(image, c1, (np.float32(c3[0]), np.float32(c3[1])), bbox_color, -1) #filled
+
+ cv2.putText(
+ image, bbox_mess, (c1[0], np.float32(c1[1] - 2)), cv2.FONT_HERSHEY_SIMPLEX, fontScale, (0, 0, 0),
+ bbox_thick // 2, lineType=cv2.LINE_AA
+ )
+ return image
+
+
+def get_anchors(anchors_path, tiny=False):
+ anchors = np.array(anchors_path)
+ if tiny:
+ return anchors.reshape(2, 3, 2)
+ else:
+ return anchors.reshape(3, 3, 2)
+
+
+def decode_train(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=0, XYSCALE=[1, 1, 1]):
+ conv_output = tf.reshape(conv_output, (tf.shape(conv_output)[0], output_size, output_size, 3, 5 + NUM_CLASS))
+
+ conv_raw_dxdy, conv_raw_dwdh, conv_raw_conf, conv_raw_prob = tf.split(conv_output, (2, 2, 1, NUM_CLASS), axis=-1)
+
+ xy_grid = tf.meshgrid(tf.range(output_size), tf.range(output_size))
+ xy_grid = tf.expand_dims(tf.stack(xy_grid, axis=-1), axis=2) # [gx, gy, 1, 2]
+ xy_grid = tf.tile(tf.expand_dims(xy_grid, axis=0), [tf.shape(conv_output)[0], 1, 1, 3, 1])
+
+ xy_grid = tf.cast(xy_grid, tf.float32)
+
+ pred_xy = ((tf.sigmoid(conv_raw_dxdy) * XYSCALE[i]) - 0.5 * (XYSCALE[i] - 1) + xy_grid) * \
+ STRIDES[i]
+ pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i])
+ pred_xywh = tf.concat([pred_xy, pred_wh], axis=-1)
+
+ pred_conf = tf.sigmoid(conv_raw_conf)
+ pred_prob = tf.sigmoid(conv_raw_prob)
+
+ return tf.concat([pred_xywh, pred_conf, pred_prob], axis=-1)
+
+
+def yolo4_input_processing(original_image):
+ image_data = cv2.resize(original_image, (416, 416))
+ image_data = image_data / 255.
+ images_data = []
+ for i in range(1):
+ images_data.append(image_data)
+ images_data = np.asarray(images_data).astype(np.float32)
+ batch_data = tf.constant(images_data)
+ return batch_data
+
+
+def yolo4_output_processing(feature_maps):
+ STRIDES = [8, 16, 32]
+ ANCHORS = get_anchors([12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401])
+ NUM_CLASS = 80
+ XYSCALE = [1.2, 1.1, 1.05]
+ iou_threshold = 0.45
+ score_threshold = 0.25
+
+ bbox_tensors = []
+ prob_tensors = []
+ score_thres = 0.2
+ for i, fm in enumerate(feature_maps):
+ if i == 0:
+ output_tensors = decode(fm, 416 // 8, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE)
+ elif i == 1:
+ output_tensors = decode(fm, 416 // 16, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE)
+ else:
+ output_tensors = decode(fm, 416 // 32, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE)
+ bbox_tensors.append(output_tensors[0])
+ prob_tensors.append(output_tensors[1])
+ pred_bbox = tf.concat(bbox_tensors, axis=1)
+ pred_prob = tf.concat(prob_tensors, axis=1)
+ boxes, pred_conf = filter_boxes(
+ pred_bbox, pred_prob, score_threshold=score_thres, input_shape=tf.constant([416, 416])
+ )
+ pred = {'concat': tf.concat([boxes, pred_conf], axis=-1)}
+
+ for key, value in pred.items():
+ boxes = value[:, :, 0:4]
+ pred_conf = value[:, :, 4:]
+
+ boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(
+ boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)),
+ scores=tf.reshape(pred_conf, (tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])),
+ max_output_size_per_class=50, max_total_size=50, iou_threshold=iou_threshold, score_threshold=score_threshold
+ )
+ output = [boxes.numpy(), scores.numpy(), classes.numpy(), valid_detections.numpy()]
+ return output
+
+
+def result_to_json(image, pred_bbox):
+ image_h, image_w, _ = image.shape
+ out_boxes, out_scores, out_classes, num_boxes = pred_bbox
+ class_names = {}
+ json_result = []
+ with open('model/coco.names', 'r') as data:
+ for ID, name in enumerate(data):
+ class_names[ID] = name.strip('\n')
+ nums_class = len(class_names)
+
+ for i in range(num_boxes[0]):
+ if int(out_classes[0][i]) < 0 or int(out_classes[0][i]) > nums_class: continue
+ coor = out_boxes[0][i]
+ coor[0] = int(coor[0] * image_h)
+ coor[2] = int(coor[2] * image_h)
+ coor[1] = int(coor[1] * image_w)
+ coor[3] = int(coor[3] * image_w)
+
+ score = float(out_scores[0][i])
+ class_ind = int(out_classes[0][i])
+ bbox = np.array([coor[1], coor[0], coor[3], coor[2]]).tolist() # [x1,y1,x2,y2]
+ json_result.append({'image': None, 'category_id': class_ind, 'bbox': bbox, 'score': score})
+
+ return json_result
+
+
+def draw_boxes_and_labels_to_image_with_json(image, json_result, class_list, save_name=None):
+ """Draw bboxes and class labels on image. Return the image with bboxes.
+
+ Parameters
+ -----------
+ image : numpy.array
+ The RGB image [height, width, channel].
+ json_result : list of dict
+ The object detection result with json format.
+ classes_list : list of str
+ For converting ID to string on image.
+ save_name : None or str
+ The name of image file (i.e. image.png), if None, not to save image.
+
+ Returns
+ -------
+ numpy.array
+ The saved image.
+
+ References
+ -----------
+ - OpenCV rectangle and putText.
+ - `scikit-image `__.
+
+ """
+ image_h, image_w, _ = image.shape
+ num_classes = len(class_list)
+ hsv_tuples = [(1.0 * x / num_classes, 1., 1.) for x in range(num_classes)]
+ colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
+ colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors))
+ random.seed(0)
+ random.shuffle(colors)
+ random.seed(None)
+ bbox_thick = int(0.6 * (image_h + image_w) / 600)
+ fontScale = 0.5
+
+ for bbox_info in json_result:
+ image_name = bbox_info['image']
+ category_id = bbox_info['category_id']
+ if category_id < 0 or category_id > num_classes: continue
+ bbox = bbox_info['bbox'] # the order of coordinates is [x1, y2, x2, y2]
+ score = bbox_info['score']
+
+ bbox_color = colors[category_id]
+ c1, c2 = (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3]))
+ cv2.rectangle(image, c1, c2, bbox_color, bbox_thick)
+
+ bbox_mess = '%s: %.2f' % (class_list[category_id], score)
+ t_size = cv2.getTextSize(bbox_mess, 0, fontScale, thickness=bbox_thick // 2)[0]
+ c3 = (c1[0] + t_size[0], c1[1] - t_size[1] - 3)
+ cv2.rectangle(image, c1, (np.float32(c3[0]), np.float32(c3[1])), bbox_color, -1)
+
+ cv2.putText(
+ image, bbox_mess, (c1[0], np.float32(c1[1] - 2)), cv2.FONT_HERSHEY_SIMPLEX, fontScale, (0, 0, 0),
+ bbox_thick // 2, lineType=cv2.LINE_AA
+ )
+
+ if save_name is not None:
+ save_image(image, save_name)
+
+ return image
\ No newline at end of file
diff --git a/examples/model_zoo/imagenet_classes.py b/examples/model_zoo/imagenet_classes.py
new file mode 100644
index 0000000..d13cfda
--- /dev/null
+++ b/examples/model_zoo/imagenet_classes.py
@@ -0,0 +1,1003 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+class_names = '''tench, Tinca tinca
+goldfish, Carassius auratus
+great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias
+tiger shark, Galeocerdo cuvieri
+hammerhead, hammerhead shark
+electric ray, crampfish, numbfish, torpedo
+stingray
+cock
+hen
+ostrich, Struthio camelus
+brambling, Fringilla montifringilla
+goldfinch, Carduelis carduelis
+house finch, linnet, Carpodacus mexicanus
+junco, snowbird
+indigo bunting, indigo finch, indigo bird, Passerina cyanea
+robin, American robin, Turdus migratorius
+bulbul
+jay
+magpie
+chickadee
+water ouzel, dipper
+kite
+bald eagle, American eagle, Haliaeetus leucocephalus
+vulture
+great grey owl, great gray owl, Strix nebulosa
+European fire salamander, Salamandra salamandra
+common newt, Triturus vulgaris
+eft
+spotted salamander, Ambystoma maculatum
+axolotl, mud puppy, Ambystoma mexicanum
+bullfrog, Rana catesbeiana
+tree frog, tree-frog
+tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui
+loggerhead, loggerhead turtle, Caretta caretta
+leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea
+mud turtle
+terrapin
+box turtle, box tortoise
+banded gecko
+common iguana, iguana, Iguana iguana
+American chameleon, anole, Anolis carolinensis
+whiptail, whiptail lizard
+agama
+frilled lizard, Chlamydosaurus kingi
+alligator lizard
+Gila monster, Heloderma suspectum
+green lizard, Lacerta viridis
+African chameleon, Chamaeleo chamaeleon
+Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis
+African crocodile, Nile crocodile, Crocodylus niloticus
+American alligator, Alligator mississipiensis
+triceratops
+thunder snake, worm snake, Carphophis amoenus
+ringneck snake, ring-necked snake, ring snake
+hognose snake, puff adder, sand viper
+green snake, grass snake
+king snake, kingsnake
+garter snake, grass snake
+water snake
+vine snake
+night snake, Hypsiglena torquata
+boa constrictor, Constrictor constrictor
+rock python, rock snake, Python sebae
+Indian cobra, Naja naja
+green mamba
+sea snake
+horned viper, cerastes, sand viper, horned asp, Cerastes cornutus
+diamondback, diamondback rattlesnake, Crotalus adamanteus
+sidewinder, horned rattlesnake, Crotalus cerastes
+trilobite
+harvestman, daddy longlegs, Phalangium opilio
+scorpion
+black and gold garden spider, Argiope aurantia
+barn spider, Araneus cavaticus
+garden spider, Aranea diademata
+black widow, Latrodectus mactans
+tarantula
+wolf spider, hunting spider
+tick
+centipede
+black grouse
+ptarmigan
+ruffed grouse, partridge, Bonasa umbellus
+prairie chicken, prairie grouse, prairie fowl
+peacock
+quail
+partridge
+African grey, African gray, Psittacus erithacus
+macaw
+sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita
+lorikeet
+coucal
+bee eater
+hornbill
+hummingbird
+jacamar
+toucan
+drake
+red-breasted merganser, Mergus serrator
+goose
+black swan, Cygnus atratus
+tusker
+echidna, spiny anteater, anteater
+platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus
+wallaby, brush kangaroo
+koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus
+wombat
+jellyfish
+sea anemone, anemone
+brain coral
+flatworm, platyhelminth
+nematode, nematode worm, roundworm
+conch
+snail
+slug
+sea slug, nudibranch
+chiton, coat-of-mail shell, sea cradle, polyplacophore
+chambered nautilus, pearly nautilus, nautilus
+Dungeness crab, Cancer magister
+rock crab, Cancer irroratus
+fiddler crab
+king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica
+American lobster, Northern lobster, Maine lobster, Homarus americanus
+spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish
+crayfish, crawfish, crawdad, crawdaddy
+hermit crab
+isopod
+white stork, Ciconia ciconia
+black stork, Ciconia nigra
+spoonbill
+flamingo
+little blue heron, Egretta caerulea
+American egret, great white heron, Egretta albus
+bittern
+crane
+limpkin, Aramus pictus
+European gallinule, Porphyrio porphyrio
+American coot, marsh hen, mud hen, water hen, Fulica americana
+bustard
+ruddy turnstone, Arenaria interpres
+red-backed sandpiper, dunlin, Erolia alpina
+redshank, Tringa totanus
+dowitcher
+oystercatcher, oyster catcher
+pelican
+king penguin, Aptenodytes patagonica
+albatross, mollymawk
+grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus
+killer whale, killer, orca, grampus, sea wolf, Orcinus orca
+dugong, Dugong dugon
+sea lion
+Chihuahua
+Japanese spaniel
+Maltese dog, Maltese terrier, Maltese
+Pekinese, Pekingese, Peke
+Shih-Tzu
+Blenheim spaniel
+papillon
+toy terrier
+Rhodesian ridgeback
+Afghan hound, Afghan
+basset, basset hound
+beagle
+bloodhound, sleuthhound
+bluetick
+black-and-tan coonhound
+Walker hound, Walker foxhound
+English foxhound
+redbone
+borzoi, Russian wolfhound
+Irish wolfhound
+Italian greyhound
+whippet
+Ibizan hound, Ibizan Podenco
+Norwegian elkhound, elkhound
+otterhound, otter hound
+Saluki, gazelle hound
+Scottish deerhound, deerhound
+Weimaraner
+Staffordshire bullterrier, Staffordshire bull terrier
+American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier
+Bedlington terrier
+Border terrier
+Kerry blue terrier
+Irish terrier
+Norfolk terrier
+Norwich terrier
+Yorkshire terrier
+wire-haired fox terrier
+Lakeland terrier
+Sealyham terrier, Sealyham
+Airedale, Airedale terrier
+cairn, cairn terrier
+Australian terrier
+Dandie Dinmont, Dandie Dinmont terrier
+Boston bull, Boston terrier
+miniature schnauzer
+giant schnauzer
+standard schnauzer
+Scotch terrier, Scottish terrier, Scottie
+Tibetan terrier, chrysanthemum dog
+silky terrier, Sydney silky
+soft-coated wheaten terrier
+West Highland white terrier
+Lhasa, Lhasa apso
+flat-coated retriever
+curly-coated retriever
+golden retriever
+Labrador retriever
+Chesapeake Bay retriever
+German short-haired pointer
+vizsla, Hungarian pointer
+English setter
+Irish setter, red setter
+Gordon setter
+Brittany spaniel
+clumber, clumber spaniel
+English springer, English springer spaniel
+Welsh springer spaniel
+cocker spaniel, English cocker spaniel, cocker
+Sussex spaniel
+Irish water spaniel
+kuvasz
+schipperke
+groenendael
+malinois
+briard
+kelpie
+komondor
+Old English sheepdog, bobtail
+Shetland sheepdog, Shetland sheep dog, Shetland
+collie
+Border collie
+Bouvier des Flandres, Bouviers des Flandres
+Rottweiler
+German shepherd, German shepherd dog, German police dog, alsatian
+Doberman, Doberman pinscher
+miniature pinscher
+Greater Swiss Mountain dog
+Bernese mountain dog
+Appenzeller
+EntleBucher
+boxer
+bull mastiff
+Tibetan mastiff
+French bulldog
+Great Dane
+Saint Bernard, St Bernard
+Eskimo dog, husky
+malamute, malemute, Alaskan malamute
+Siberian husky
+dalmatian, coach dog, carriage dog
+affenpinscher, monkey pinscher, monkey dog
+basenji
+pug, pug-dog
+Leonberg
+Newfoundland, Newfoundland dog
+Great Pyrenees
+Samoyed, Samoyede
+Pomeranian
+chow, chow chow
+keeshond
+Brabancon griffon
+Pembroke, Pembroke Welsh corgi
+Cardigan, Cardigan Welsh corgi
+toy poodle
+miniature poodle
+standard poodle
+Mexican hairless
+timber wolf, grey wolf, gray wolf, Canis lupus
+white wolf, Arctic wolf, Canis lupus tundrarum
+red wolf, maned wolf, Canis rufus, Canis niger
+coyote, prairie wolf, brush wolf, Canis latrans
+dingo, warrigal, warragal, Canis dingo
+dhole, Cuon alpinus
+African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus
+hyena, hyaena
+red fox, Vulpes vulpes
+kit fox, Vulpes macrotis
+Arctic fox, white fox, Alopex lagopus
+grey fox, gray fox, Urocyon cinereoargenteus
+tabby, tabby cat
+tiger cat
+Persian cat
+Siamese cat, Siamese
+Egyptian cat
+cougar, puma, catamount, mountain lion, painter, panther, Felis concolor
+lynx, catamount
+leopard, Panthera pardus
+snow leopard, ounce, Panthera uncia
+jaguar, panther, Panthera onca, Felis onca
+lion, king of beasts, Panthera leo
+tiger, Panthera tigris
+cheetah, chetah, Acinonyx jubatus
+brown bear, bruin, Ursus arctos
+American black bear, black bear, Ursus americanus, Euarctos americanus
+ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus
+sloth bear, Melursus ursinus, Ursus ursinus
+mongoose
+meerkat, mierkat
+tiger beetle
+ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle
+ground beetle, carabid beetle
+long-horned beetle, longicorn, longicorn beetle
+leaf beetle, chrysomelid
+dung beetle
+rhinoceros beetle
+weevil
+fly
+bee
+ant, emmet, pismire
+grasshopper, hopper
+cricket
+walking stick, walkingstick, stick insect
+cockroach, roach
+mantis, mantid
+cicada, cicala
+leafhopper
+lacewing, lacewing fly
+dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk
+damselfly
+admiral
+ringlet, ringlet butterfly
+monarch, monarch butterfly, milkweed butterfly, Danaus plexippus
+cabbage butterfly
+sulphur butterfly, sulfur butterfly
+lycaenid, lycaenid butterfly
+starfish, sea star
+sea urchin
+sea cucumber, holothurian
+wood rabbit, cottontail, cottontail rabbit
+hare
+Angora, Angora rabbit
+hamster
+porcupine, hedgehog
+fox squirrel, eastern fox squirrel, Sciurus niger
+marmot
+beaver
+guinea pig, Cavia cobaya
+sorrel
+zebra
+hog, pig, grunter, squealer, Sus scrofa
+wild boar, boar, Sus scrofa
+warthog
+hippopotamus, hippo, river horse, Hippopotamus amphibius
+ox
+water buffalo, water ox, Asiatic buffalo, Bubalus bubalis
+bison
+ram, tup
+bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis
+ibex, Capra ibex
+hartebeest
+impala, Aepyceros melampus
+gazelle
+Arabian camel, dromedary, Camelus dromedarius
+llama
+weasel
+mink
+polecat, fitch, foulmart, foumart, Mustela putorius
+black-footed ferret, ferret, Mustela nigripes
+otter
+skunk, polecat, wood pussy
+badger
+armadillo
+three-toed sloth, ai, Bradypus tridactylus
+orangutan, orang, orangutang, Pongo pygmaeus
+gorilla, Gorilla gorilla
+chimpanzee, chimp, Pan troglodytes
+gibbon, Hylobates lar
+siamang, Hylobates syndactylus, Symphalangus syndactylus
+guenon, guenon monkey
+patas, hussar monkey, Erythrocebus patas
+baboon
+macaque
+langur
+colobus, colobus monkey
+proboscis monkey, Nasalis larvatus
+marmoset
+capuchin, ringtail, Cebus capucinus
+howler monkey, howler
+titi, titi monkey
+spider monkey, Ateles geoffroyi
+squirrel monkey, Saimiri sciureus
+Madagascar cat, ring-tailed lemur, Lemur catta
+indri, indris, Indri indri, Indri brevicaudatus
+Indian elephant, Elephas maximus
+African elephant, Loxodonta africana
+lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens
+giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca
+barracouta, snoek
+eel
+coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch
+rock beauty, Holocanthus tricolor
+anemone fish
+sturgeon
+gar, garfish, garpike, billfish, Lepisosteus osseus
+lionfish
+puffer, pufferfish, blowfish, globefish
+abacus
+abaya
+academic gown, academic robe, judge's robe
+accordion, piano accordion, squeeze box
+acoustic guitar
+aircraft carrier, carrier, flattop, attack aircraft carrier
+airliner
+airship, dirigible
+altar
+ambulance
+amphibian, amphibious vehicle
+analog clock
+apiary, bee house
+apron
+ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin
+assault rifle, assault gun
+backpack, back pack, knapsack, packsack, rucksack, haversack
+bakery, bakeshop, bakehouse
+balance beam, beam
+balloon
+ballpoint, ballpoint pen, ballpen, Biro
+Band Aid
+banjo
+bannister, banister, balustrade, balusters, handrail
+barbell
+barber chair
+barbershop
+barn
+barometer
+barrel, cask
+barrow, garden cart, lawn cart, wheelbarrow
+baseball
+basketball
+bassinet
+bassoon
+bathing cap, swimming cap
+bath towel
+bathtub, bathing tub, bath, tub
+beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon
+beacon, lighthouse, beacon light, pharos
+beaker
+bearskin, busby, shako
+beer bottle
+beer glass
+bell cote, bell cot
+bib
+bicycle-built-for-two, tandem bicycle, tandem
+bikini, two-piece
+binder, ring-binder
+binoculars, field glasses, opera glasses
+birdhouse
+boathouse
+bobsled, bobsleigh, bob
+bolo tie, bolo, bola tie, bola
+bonnet, poke bonnet
+bookcase
+bookshop, bookstore, bookstall
+bottlecap
+bow
+bow tie, bow-tie, bowtie
+brass, memorial tablet, plaque
+brassiere, bra, bandeau
+breakwater, groin, groyne, mole, bulwark, seawall, jetty
+breastplate, aegis, egis
+broom
+bucket, pail
+buckle
+bulletproof vest
+bullet train, bullet
+butcher shop, meat market
+cab, hack, taxi, taxicab
+caldron, cauldron
+candle, taper, wax light
+cannon
+canoe
+can opener, tin opener
+cardigan
+car mirror
+carousel, carrousel, merry-go-round, roundabout, whirligig
+carpenter's kit, tool kit
+carton
+car wheel
+cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM
+cassette
+cassette player
+castle
+catamaran
+CD player
+cello, violoncello
+cellular telephone, cellular phone, cellphone, cell, mobile phone
+chain
+chainlink fence
+chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour
+chain saw, chainsaw
+chest
+chiffonier, commode
+chime, bell, gong
+china cabinet, china closet
+Christmas stocking
+church, church building
+cinema, movie theater, movie theatre, movie house, picture palace
+cleaver, meat cleaver, chopper
+cliff dwelling
+cloak
+clog, geta, patten, sabot
+cocktail shaker
+coffee mug
+coffeepot
+coil, spiral, volute, whorl, helix
+combination lock
+computer keyboard, keypad
+confectionery, confectionary, candy store
+container ship, containership, container vessel
+convertible
+corkscrew, bottle screw
+cornet, horn, trumpet, trump
+cowboy boot
+cowboy hat, ten-gallon hat
+cradle
+crane
+crash helmet
+crate
+crib, cot
+Crock Pot
+croquet ball
+crutch
+cuirass
+dam, dike, dyke
+desk
+desktop computer
+dial telephone, dial phone
+diaper, nappy, napkin
+digital clock
+digital watch
+dining table, board
+dishrag, dishcloth
+dishwasher, dish washer, dishwashing machine
+disk brake, disc brake
+dock, dockage, docking facility
+dogsled, dog sled, dog sleigh
+dome
+doormat, welcome mat
+drilling platform, offshore rig
+drum, membranophone, tympan
+drumstick
+dumbbell
+Dutch oven
+electric fan, blower
+electric guitar
+electric locomotive
+entertainment center
+envelope
+espresso maker
+face powder
+feather boa, boa
+file, file cabinet, filing cabinet
+fireboat
+fire engine, fire truck
+fire screen, fireguard
+flagpole, flagstaff
+flute, transverse flute
+folding chair
+football helmet
+forklift
+fountain
+fountain pen
+four-poster
+freight car
+French horn, horn
+frying pan, frypan, skillet
+fur coat
+garbage truck, dustcart
+gasmask, respirator, gas helmet
+gas pump, gasoline pump, petrol pump, island dispenser
+goblet
+go-kart
+golf ball
+golfcart, golf cart
+gondola
+gong, tam-tam
+gown
+grand piano, grand
+greenhouse, nursery, glasshouse
+grille, radiator grille
+grocery store, grocery, food market, market
+guillotine
+hair slide
+hair spray
+half track
+hammer
+hamper
+hand blower, blow dryer, blow drier, hair dryer, hair drier
+hand-held computer, hand-held microcomputer
+handkerchief, hankie, hanky, hankey
+hard disc, hard disk, fixed disk
+harmonica, mouth organ, harp, mouth harp
+harp
+harvester, reaper
+hatchet
+holster
+home theater, home theatre
+honeycomb
+hook, claw
+hoopskirt, crinoline
+horizontal bar, high bar
+horse cart, horse-cart
+hourglass
+iPod
+iron, smoothing iron
+jack-o'-lantern
+jean, blue jean, denim
+jeep, landrover
+jersey, T-shirt, tee shirt
+jigsaw puzzle
+jinrikisha, ricksha, rickshaw
+joystick
+kimono
+knee pad
+knot
+lab coat, laboratory coat
+ladle
+lampshade, lamp shade
+laptop, laptop computer
+lawn mower, mower
+lens cap, lens cover
+letter opener, paper knife, paperknife
+library
+lifeboat
+lighter, light, igniter, ignitor
+limousine, limo
+liner, ocean liner
+lipstick, lip rouge
+Loafer
+lotion
+loudspeaker, speaker, speaker unit, loudspeaker system, speaker system
+loupe, jeweler's loupe
+lumbermill, sawmill
+magnetic compass
+mailbag, postbag
+mailbox, letter box
+maillot
+maillot, tank suit
+manhole cover
+maraca
+marimba, xylophone
+mask
+matchstick
+maypole
+maze, labyrinth
+measuring cup
+medicine chest, medicine cabinet
+megalith, megalithic structure
+microphone, mike
+microwave, microwave oven
+military uniform
+milk can
+minibus
+miniskirt, mini
+minivan
+missile
+mitten
+mixing bowl
+mobile home, manufactured home
+Model T
+modem
+monastery
+monitor
+moped
+mortar
+mortarboard
+mosque
+mosquito net
+motor scooter, scooter
+mountain bike, all-terrain bike, off-roader
+mountain tent
+mouse, computer mouse
+mousetrap
+moving van
+muzzle
+nail
+neck brace
+necklace
+nipple
+notebook, notebook computer
+obelisk
+oboe, hautboy, hautbois
+ocarina, sweet potato
+odometer, hodometer, mileometer, milometer
+oil filter
+organ, pipe organ
+oscilloscope, scope, cathode-ray oscilloscope, CRO
+overskirt
+oxcart
+oxygen mask
+packet
+paddle, boat paddle
+paddlewheel, paddle wheel
+padlock
+paintbrush
+pajama, pyjama, pj's, jammies
+palace
+panpipe, pandean pipe, syrinx
+paper towel
+parachute, chute
+parallel bars, bars
+park bench
+parking meter
+passenger car, coach, carriage
+patio, terrace
+pay-phone, pay-station
+pedestal, plinth, footstall
+pencil box, pencil case
+pencil sharpener
+perfume, essence
+Petri dish
+photocopier
+pick, plectrum, plectron
+pickelhaube
+picket fence, paling
+pickup, pickup truck
+pier
+piggy bank, penny bank
+pill bottle
+pillow
+ping-pong ball
+pinwheel
+pirate, pirate ship
+pitcher, ewer
+plane, carpenter's plane, woodworking plane
+planetarium
+plastic bag
+plate rack
+plow, plough
+plunger, plumber's helper
+Polaroid camera, Polaroid Land camera
+pole
+police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria
+poncho
+pool table, billiard table, snooker table
+pop bottle, soda bottle
+pot, flowerpot
+potter's wheel
+power drill
+prayer rug, prayer mat
+printer
+prison, prison house
+projectile, missile
+projector
+puck, hockey puck
+punching bag, punch bag, punching ball, punchball
+purse
+quill, quill pen
+quilt, comforter, comfort, puff
+racer, race car, racing car
+racket, racquet
+radiator
+radio, wireless
+radio telescope, radio reflector
+rain barrel
+recreational vehicle, RV, R.V.
+reel
+reflex camera
+refrigerator, icebox
+remote control, remote
+restaurant, eating house, eating place, eatery
+revolver, six-gun, six-shooter
+rifle
+rocking chair, rocker
+rotisserie
+rubber eraser, rubber, pencil eraser
+rugby ball
+rule, ruler
+running shoe
+safe
+safety pin
+saltshaker, salt shaker
+sandal
+sarong
+sax, saxophone
+scabbard
+scale, weighing machine
+school bus
+schooner
+scoreboard
+screen, CRT screen
+screw
+screwdriver
+seat belt, seatbelt
+sewing machine
+shield, buckler
+shoe shop, shoe-shop, shoe store
+shoji
+shopping basket
+shopping cart
+shovel
+shower cap
+shower curtain
+ski
+ski mask
+sleeping bag
+slide rule, slipstick
+sliding door
+slot, one-armed bandit
+snorkel
+snowmobile
+snowplow, snowplough
+soap dispenser
+soccer ball
+sock
+solar dish, solar collector, solar furnace
+sombrero
+soup bowl
+space bar
+space heater
+space shuttle
+spatula
+speedboat
+spider web, spider's web
+spindle
+sports car, sport car
+spotlight, spot
+stage
+steam locomotive
+steel arch bridge
+steel drum
+stethoscope
+stole
+stone wall
+stopwatch, stop watch
+stove
+strainer
+streetcar, tram, tramcar, trolley, trolley car
+stretcher
+studio couch, day bed
+stupa, tope
+submarine, pigboat, sub, U-boat
+suit, suit of clothes
+sundial
+sunglass
+sunglasses, dark glasses, shades
+sunscreen, sunblock, sun blocker
+suspension bridge
+swab, swob, mop
+sweatshirt
+swimming trunks, bathing trunks
+swing
+switch, electric switch, electrical switch
+syringe
+table lamp
+tank, army tank, armored combat vehicle, armoured combat vehicle
+tape player
+teapot
+teddy, teddy bear
+television, television system
+tennis ball
+thatch, thatched roof
+theater curtain, theatre curtain
+thimble
+thresher, thrasher, threshing machine
+throne
+tile roof
+toaster
+tobacco shop, tobacconist shop, tobacconist
+toilet seat
+torch
+totem pole
+tow truck, tow car, wrecker
+toyshop
+tractor
+trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi
+tray
+trench coat
+tricycle, trike, velocipede
+trimaran
+tripod
+triumphal arch
+trolleybus, trolley coach, trackless trolley
+trombone
+tub, vat
+turnstile
+typewriter keyboard
+umbrella
+unicycle, monocycle
+upright, upright piano
+vacuum, vacuum cleaner
+vase
+vault
+velvet
+vending machine
+vestment
+viaduct
+violin, fiddle
+volleyball
+waffle iron
+wall clock
+wallet, billfold, notecase, pocketbook
+wardrobe, closet, press
+warplane, military plane
+washbasin, handbasin, washbowl, lavabo, wash-hand basin
+washer, automatic washer, washing machine
+water bottle
+water jug
+water tower
+whiskey jug
+whistle
+wig
+window screen
+window shade
+Windsor tie
+wine bottle
+wing
+wok
+wooden spoon
+wool, woolen, woollen
+worm fence, snake fence, snake-rail fence, Virginia fence
+wreck
+yawl
+yurt
+web site, website, internet site, site
+comic book
+crossword puzzle, crossword
+street sign
+traffic light, traffic signal, stoplight
+book jacket, dust cover, dust jacket, dust wrapper
+menu
+plate
+guacamole
+consomme
+hot pot, hotpot
+trifle
+ice cream, icecream
+ice lolly, lolly, lollipop, popsicle
+French loaf
+bagel, beigel
+pretzel
+cheeseburger
+hotdog, hot dog, red hot
+mashed potato
+head cabbage
+broccoli
+cauliflower
+zucchini, courgette
+spaghetti squash
+acorn squash
+butternut squash
+cucumber, cuke
+artichoke, globe artichoke
+bell pepper
+cardoon
+mushroom
+Granny Smith
+strawberry
+orange
+lemon
+fig
+pineapple, ananas
+banana
+jackfruit, jak, jack
+custard apple
+pomegranate
+hay
+carbonara
+chocolate sauce, chocolate syrup
+dough
+meat loaf, meatloaf
+pizza, pizza pie
+potpie
+burrito
+red wine
+espresso
+cup
+eggnog
+alp
+bubble
+cliff, drop, drop-off
+coral reef
+geyser
+lakeside, lakeshore
+promontory, headland, head, foreland
+sandbar, sand bar
+seashore, coast, seacoast, sea-coast
+valley, vale
+volcano
+ballplayer, baseball player
+groom, bridegroom
+scuba diver
+rapeseed
+daisy
+yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum
+corn
+acorn
+hip, rose hip, rosehip
+buckeye, horse chestnut, conker
+coral fungus
+agaric
+gyromitra
+stinkhorn, carrion fungus
+earthstar
+hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa
+bolete
+ear, spike, capitulum
+toilet tissue, toilet paper, bathroom tissue'''.split("\n")
diff --git a/examples/model_zoo/model/coco.names b/examples/model_zoo/model/coco.names
new file mode 100644
index 0000000..ec82f0f
--- /dev/null
+++ b/examples/model_zoo/model/coco.names
@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorbike
+aeroplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+sofa
+potted plant
+bed
+dining table
+toilet
+tvmonitor
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush
diff --git a/examples/model_zoo/model/weights_2.txt b/examples/model_zoo/model/weights_2.txt
new file mode 100644
index 0000000..42cc499
--- /dev/null
+++ b/examples/model_zoo/model/weights_2.txt
@@ -0,0 +1,541 @@
+conv2d_1/filters:0
+batchnorm2d_1/beta:0
+batchnorm2d_1/gamma:0
+batchnorm2d_1/moving_mean:0
+batchnorm2d_1/moving_var:0
+conv2d_2/filters:0
+batchnorm2d_2/beta:0
+batchnorm2d_2/gamma:0
+batchnorm2d_2/moving_mean:0
+batchnorm2d_2/moving_var:0
+conv_rote_block_1/filters:0
+conv2d_3/filters:0
+batchnorm2d_3/beta:0
+batchnorm2d_3/gamma:0
+batchnorm2d_3/moving_mean:0
+batchnorm2d_3/moving_var:0
+batchnorm2d_4/beta:0
+batchnorm2d_4/gamma:0
+batchnorm2d_4/moving_mean:0
+batchnorm2d_4/moving_var:0
+conv2d_4/filters:0
+batchnorm2d_5/beta:0
+batchnorm2d_5/gamma:0
+batchnorm2d_5/moving_mean:0
+batchnorm2d_5/moving_var:0
+conv2d_5/filters:0
+batchnorm2d_6/beta:0
+batchnorm2d_6/gamma:0
+batchnorm2d_6/moving_mean:0
+batchnorm2d_6/moving_var:0
+conv2d_6/filters:0
+batchnorm2d_7/beta:0
+batchnorm2d_7/gamma:0
+batchnorm2d_7/moving_mean:0
+batchnorm2d_7/moving_var:0
+conv2d_7/filters:0
+batchnorm2d_8/beta:0
+batchnorm2d_8/gamma:0
+batchnorm2d_8/moving_mean:0
+batchnorm2d_8/moving_var:0
+conv2d_8/filters:0
+batchnorm2d_9/beta:0
+batchnorm2d_9/gamma:0
+batchnorm2d_9/moving_mean:0
+batchnorm2d_9/moving_var:0
+conv_rote_block_2/filters:0
+conv2d_9/filters:0
+batchnorm2d_10/beta:0
+batchnorm2d_10/gamma:0
+batchnorm2d_10/moving_mean:0
+batchnorm2d_10/moving_var:0
+batchnorm2d_11/beta:0
+batchnorm2d_11/gamma:0
+batchnorm2d_11/moving_mean:0
+batchnorm2d_11/moving_var:0
+conv2d_10/filters:0
+batchnorm2d_12/beta:0
+batchnorm2d_12/gamma:0
+batchnorm2d_12/moving_mean:0
+batchnorm2d_12/moving_var:0
+conv2d_11/filters:0
+batchnorm2d_13/beta:0
+batchnorm2d_13/gamma:0
+batchnorm2d_13/moving_mean:0
+batchnorm2d_13/moving_var:0
+conv2d_12/filters:0
+batchnorm2d_14/beta:0
+batchnorm2d_14/gamma:0
+batchnorm2d_14/moving_mean:0
+batchnorm2d_14/moving_var:0
+conv2d_13/filters:0
+batchnorm2d_15/beta:0
+batchnorm2d_15/gamma:0
+batchnorm2d_15/moving_mean:0
+batchnorm2d_15/moving_var:0
+conv2d_14/filters:0
+batchnorm2d_16/beta:0
+batchnorm2d_16/gamma:0
+batchnorm2d_16/moving_mean:0
+batchnorm2d_16/moving_var:0
+conv2d_15/filters:0
+batchnorm2d_17/beta:0
+batchnorm2d_17/gamma:0
+batchnorm2d_17/moving_mean:0
+batchnorm2d_17/moving_var:0
+conv2d_16/filters:0
+batchnorm2d_18/beta:0
+batchnorm2d_18/gamma:0
+batchnorm2d_18/moving_mean:0
+batchnorm2d_18/moving_var:0
+conv_rote_block_3/filters:0
+conv2d_17/filters:0
+batchnorm2d_19/beta:0
+batchnorm2d_19/gamma:0
+batchnorm2d_19/moving_mean:0
+batchnorm2d_19/moving_var:0
+batchnorm2d_20/beta:0
+batchnorm2d_20/gamma:0
+batchnorm2d_20/moving_mean:0
+batchnorm2d_20/moving_var:0
+conv2d_18/filters:0
+batchnorm2d_21/beta:0
+batchnorm2d_21/gamma:0
+batchnorm2d_21/moving_mean:0
+batchnorm2d_21/moving_var:0
+conv2d_19/filters:0
+batchnorm2d_22/beta:0
+batchnorm2d_22/gamma:0
+batchnorm2d_22/moving_mean:0
+batchnorm2d_22/moving_var:0
+conv2d_20/filters:0
+batchnorm2d_23/beta:0
+batchnorm2d_23/gamma:0
+batchnorm2d_23/moving_mean:0
+batchnorm2d_23/moving_var:0
+conv2d_21/filters:0
+batchnorm2d_24/beta:0
+batchnorm2d_24/gamma:0
+batchnorm2d_24/moving_mean:0
+batchnorm2d_24/moving_var:0
+conv2d_22/filters:0
+batchnorm2d_25/beta:0
+batchnorm2d_25/gamma:0
+batchnorm2d_25/moving_mean:0
+batchnorm2d_25/moving_var:0
+conv2d_23/filters:0
+batchnorm2d_26/beta:0
+batchnorm2d_26/gamma:0
+batchnorm2d_26/moving_mean:0
+batchnorm2d_26/moving_var:0
+conv2d_24/filters:0
+batchnorm2d_27/beta:0
+batchnorm2d_27/gamma:0
+batchnorm2d_27/moving_mean:0
+batchnorm2d_27/moving_var:0
+conv2d_25/filters:0
+batchnorm2d_28/beta:0
+batchnorm2d_28/gamma:0
+batchnorm2d_28/moving_mean:0
+batchnorm2d_28/moving_var:0
+conv2d_26/filters:0
+batchnorm2d_29/beta:0
+batchnorm2d_29/gamma:0
+batchnorm2d_29/moving_mean:0
+batchnorm2d_29/moving_var:0
+conv2d_27/filters:0
+batchnorm2d_30/beta:0
+batchnorm2d_30/gamma:0
+batchnorm2d_30/moving_mean:0
+batchnorm2d_30/moving_var:0
+conv2d_28/filters:0
+batchnorm2d_31/beta:0
+batchnorm2d_31/gamma:0
+batchnorm2d_31/moving_mean:0
+batchnorm2d_31/moving_var:0
+conv2d_29/filters:0
+batchnorm2d_32/beta:0
+batchnorm2d_32/gamma:0
+batchnorm2d_32/moving_mean:0
+batchnorm2d_32/moving_var:0
+conv2d_30/filters:0
+batchnorm2d_33/beta:0
+batchnorm2d_33/gamma:0
+batchnorm2d_33/moving_mean:0
+batchnorm2d_33/moving_var:0
+conv2d_31/filters:0
+batchnorm2d_34/beta:0
+batchnorm2d_34/gamma:0
+batchnorm2d_34/moving_mean:0
+batchnorm2d_34/moving_var:0
+conv2d_32/filters:0
+batchnorm2d_35/beta:0
+batchnorm2d_35/gamma:0
+batchnorm2d_35/moving_mean:0
+batchnorm2d_35/moving_var:0
+conv2d_33/filters:0
+batchnorm2d_36/beta:0
+batchnorm2d_36/gamma:0
+batchnorm2d_36/moving_mean:0
+batchnorm2d_36/moving_var:0
+conv2d_34/filters:0
+batchnorm2d_37/beta:0
+batchnorm2d_37/gamma:0
+batchnorm2d_37/moving_mean:0
+batchnorm2d_37/moving_var:0
+conv2d_35/filters:0
+batchnorm2d_38/beta:0
+batchnorm2d_38/gamma:0
+batchnorm2d_38/moving_mean:0
+batchnorm2d_38/moving_var:0
+conv_yolo_2/filters:0
+batchnorm2d_87/beta:0
+batchnorm2d_87/gamma:0
+batchnorm2d_87/moving_mean:0
+batchnorm2d_87/moving_var:0
+conv2d_36/filters:0
+batchnorm2d_39/beta:0
+batchnorm2d_39/gamma:0
+batchnorm2d_39/moving_mean:0
+batchnorm2d_39/moving_var:0
+conv_rote_block_4/filters:0
+conv2d_37/filters:0
+batchnorm2d_40/beta:0
+batchnorm2d_40/gamma:0
+batchnorm2d_40/moving_mean:0
+batchnorm2d_40/moving_var:0
+batchnorm2d_41/beta:0
+batchnorm2d_41/gamma:0
+batchnorm2d_41/moving_mean:0
+batchnorm2d_41/moving_var:0
+conv2d_38/filters:0
+batchnorm2d_42/beta:0
+batchnorm2d_42/gamma:0
+batchnorm2d_42/moving_mean:0
+batchnorm2d_42/moving_var:0
+conv2d_39/filters:0
+batchnorm2d_43/beta:0
+batchnorm2d_43/gamma:0
+batchnorm2d_43/moving_mean:0
+batchnorm2d_43/moving_var:0
+conv2d_40/filters:0
+batchnorm2d_44/beta:0
+batchnorm2d_44/gamma:0
+batchnorm2d_44/moving_mean:0
+batchnorm2d_44/moving_var:0
+conv2d_41/filters:0
+batchnorm2d_45/beta:0
+batchnorm2d_45/gamma:0
+batchnorm2d_45/moving_mean:0
+batchnorm2d_45/moving_var:0
+conv2d_42/filters:0
+batchnorm2d_46/beta:0
+batchnorm2d_46/gamma:0
+batchnorm2d_46/moving_mean:0
+batchnorm2d_46/moving_var:0
+conv2d_43/filters:0
+batchnorm2d_47/beta:0
+batchnorm2d_47/gamma:0
+batchnorm2d_47/moving_mean:0
+batchnorm2d_47/moving_var:0
+conv2d_44/filters:0
+batchnorm2d_48/beta:0
+batchnorm2d_48/gamma:0
+batchnorm2d_48/moving_mean:0
+batchnorm2d_48/moving_var:0
+conv2d_45/filters:0
+batchnorm2d_49/beta:0
+batchnorm2d_49/gamma:0
+batchnorm2d_49/moving_mean:0
+batchnorm2d_49/moving_var:0
+conv2d_46/filters:0
+batchnorm2d_50/beta:0
+batchnorm2d_50/gamma:0
+batchnorm2d_50/moving_mean:0
+batchnorm2d_50/moving_var:0
+conv2d_47/filters:0
+batchnorm2d_51/beta:0
+batchnorm2d_51/gamma:0
+batchnorm2d_51/moving_mean:0
+batchnorm2d_51/moving_var:0
+conv2d_48/filters:0
+batchnorm2d_52/beta:0
+batchnorm2d_52/gamma:0
+batchnorm2d_52/moving_mean:0
+batchnorm2d_52/moving_var:0
+conv2d_49/filters:0
+batchnorm2d_53/beta:0
+batchnorm2d_53/gamma:0
+batchnorm2d_53/moving_mean:0
+batchnorm2d_53/moving_var:0
+conv2d_50/filters:0
+batchnorm2d_54/beta:0
+batchnorm2d_54/gamma:0
+batchnorm2d_54/moving_mean:0
+batchnorm2d_54/moving_var:0
+conv2d_51/filters:0
+batchnorm2d_55/beta:0
+batchnorm2d_55/gamma:0
+batchnorm2d_55/moving_mean:0
+batchnorm2d_55/moving_var:0
+conv2d_52/filters:0
+batchnorm2d_56/beta:0
+batchnorm2d_56/gamma:0
+batchnorm2d_56/moving_mean:0
+batchnorm2d_56/moving_var:0
+conv2d_53/filters:0
+batchnorm2d_57/beta:0
+batchnorm2d_57/gamma:0
+batchnorm2d_57/moving_mean:0
+batchnorm2d_57/moving_var:0
+conv2d_54/filters:0
+batchnorm2d_58/beta:0
+batchnorm2d_58/gamma:0
+batchnorm2d_58/moving_mean:0
+batchnorm2d_58/moving_var:0
+conv2d_55/filters:0
+batchnorm2d_59/beta:0
+batchnorm2d_59/gamma:0
+batchnorm2d_59/moving_mean:0
+batchnorm2d_59/moving_var:0
+conv_yolo_1/filters:0
+batchnorm2d_80/beta:0
+batchnorm2d_80/gamma:0
+batchnorm2d_80/moving_mean:0
+batchnorm2d_80/moving_var:0
+conv2d_56/filters:0
+batchnorm2d_60/beta:0
+batchnorm2d_60/gamma:0
+batchnorm2d_60/moving_mean:0
+batchnorm2d_60/moving_var:0
+conv_rote_block_5/filters:0
+conv2d_57/filters:0
+batchnorm2d_61/beta:0
+batchnorm2d_61/gamma:0
+batchnorm2d_61/moving_mean:0
+batchnorm2d_61/moving_var:0
+batchnorm2d_62/beta:0
+batchnorm2d_62/gamma:0
+batchnorm2d_62/moving_mean:0
+batchnorm2d_62/moving_var:0
+conv2d_58/filters:0
+batchnorm2d_63/beta:0
+batchnorm2d_63/gamma:0
+batchnorm2d_63/moving_mean:0
+batchnorm2d_63/moving_var:0
+conv2d_59/filters:0
+batchnorm2d_64/beta:0
+batchnorm2d_64/gamma:0
+batchnorm2d_64/moving_mean:0
+batchnorm2d_64/moving_var:0
+conv2d_60/filters:0
+batchnorm2d_65/beta:0
+batchnorm2d_65/gamma:0
+batchnorm2d_65/moving_mean:0
+batchnorm2d_65/moving_var:0
+conv2d_61/filters:0
+batchnorm2d_66/beta:0
+batchnorm2d_66/gamma:0
+batchnorm2d_66/moving_mean:0
+batchnorm2d_66/moving_var:0
+conv2d_62/filters:0
+batchnorm2d_67/beta:0
+batchnorm2d_67/gamma:0
+batchnorm2d_67/moving_mean:0
+batchnorm2d_67/moving_var:0
+conv2d_63/filters:0
+batchnorm2d_68/beta:0
+batchnorm2d_68/gamma:0
+batchnorm2d_68/moving_mean:0
+batchnorm2d_68/moving_var:0
+conv2d_64/filters:0
+batchnorm2d_69/beta:0
+batchnorm2d_69/gamma:0
+batchnorm2d_69/moving_mean:0
+batchnorm2d_69/moving_var:0
+conv2d_65/filters:0
+batchnorm2d_70/beta:0
+batchnorm2d_70/gamma:0
+batchnorm2d_70/moving_mean:0
+batchnorm2d_70/moving_var:0
+conv2d_66/filters:0
+batchnorm2d_71/beta:0
+batchnorm2d_71/gamma:0
+batchnorm2d_71/moving_mean:0
+batchnorm2d_71/moving_var:0
+conv2d_67/filters:0
+batchnorm2d_72/beta:0
+batchnorm2d_72/gamma:0
+batchnorm2d_72/moving_mean:0
+batchnorm2d_72/moving_var:0
+conv2d_68/filters:0
+batchnorm2d_73/beta:0
+batchnorm2d_73/gamma:0
+batchnorm2d_73/moving_mean:0
+batchnorm2d_73/moving_var:0
+conv2d_69/filters:0
+batchnorm2d_74/beta:0
+batchnorm2d_74/gamma:0
+batchnorm2d_74/moving_mean:0
+batchnorm2d_74/moving_var:0
+conv2d_70/filters:0
+batchnorm2d_75/beta:0
+batchnorm2d_75/gamma:0
+batchnorm2d_75/moving_mean:0
+batchnorm2d_75/moving_var:0
+conv2d_71/filters:0
+batchnorm2d_76/beta:0
+batchnorm2d_76/gamma:0
+batchnorm2d_76/moving_mean:0
+batchnorm2d_76/moving_var:0
+conv2d_72/filters:0
+batchnorm2d_77/beta:0
+batchnorm2d_77/gamma:0
+batchnorm2d_77/moving_mean:0
+batchnorm2d_77/moving_var:0
+conv2d_73/filters:0
+batchnorm2d_78/beta:0
+batchnorm2d_78/gamma:0
+batchnorm2d_78/moving_mean:0
+batchnorm2d_78/moving_var:0
+conv2d_74/filters:0
+batchnorm2d_79/beta:0
+batchnorm2d_79/gamma:0
+batchnorm2d_79/moving_mean:0
+batchnorm2d_79/moving_var:0
+conv2d_75/filters:0
+batchnorm2d_81/beta:0
+batchnorm2d_81/gamma:0
+batchnorm2d_81/moving_mean:0
+batchnorm2d_81/moving_var:0
+conv2d_76/filters:0
+batchnorm2d_82/beta:0
+batchnorm2d_82/gamma:0
+batchnorm2d_82/moving_mean:0
+batchnorm2d_82/moving_var:0
+conv2d_77/filters:0
+batchnorm2d_83/beta:0
+batchnorm2d_83/gamma:0
+batchnorm2d_83/moving_mean:0
+batchnorm2d_83/moving_var:0
+conv2d_78/filters:0
+batchnorm2d_84/beta:0
+batchnorm2d_84/gamma:0
+batchnorm2d_84/moving_mean:0
+batchnorm2d_84/moving_var:0
+conv2d_79/filters:0
+batchnorm2d_85/beta:0
+batchnorm2d_85/gamma:0
+batchnorm2d_85/moving_mean:0
+batchnorm2d_85/moving_var:0
+conv2d_80/filters:0
+batchnorm2d_86/beta:0
+batchnorm2d_86/gamma:0
+batchnorm2d_86/moving_mean:0
+batchnorm2d_86/moving_var:0
+conv2d_81/filters:0
+batchnorm2d_88/beta:0
+batchnorm2d_88/gamma:0
+batchnorm2d_88/moving_mean:0
+batchnorm2d_88/moving_var:0
+conv2d_82/filters:0
+batchnorm2d_89/beta:0
+batchnorm2d_89/gamma:0
+batchnorm2d_89/moving_mean:0
+batchnorm2d_89/moving_var:0
+conv2d_83/filters:0
+batchnorm2d_90/beta:0
+batchnorm2d_90/gamma:0
+batchnorm2d_90/moving_mean:0
+batchnorm2d_90/moving_var:0
+conv2d_84/filters:0
+batchnorm2d_91/beta:0
+batchnorm2d_91/gamma:0
+batchnorm2d_91/moving_mean:0
+batchnorm2d_91/moving_var:0
+conv2d_85/filters:0
+batchnorm2d_92/beta:0
+batchnorm2d_92/gamma:0
+batchnorm2d_92/moving_mean:0
+batchnorm2d_92/moving_var:0
+conv_route_1/filters:0
+batchnorm2d_93/beta:0
+batchnorm2d_93/gamma:0
+batchnorm2d_93/moving_mean:0
+batchnorm2d_93/moving_var:0
+conv_route_2/filters:0
+conv2d_86/filters:0
+conv2d_86/biases:0
+batchnorm2d_94/beta:0
+batchnorm2d_94/gamma:0
+batchnorm2d_94/moving_mean:0
+batchnorm2d_94/moving_var:0
+conv2d_87/filters:0
+batchnorm2d_95/beta:0
+batchnorm2d_95/gamma:0
+batchnorm2d_95/moving_mean:0
+batchnorm2d_95/moving_var:0
+conv2d_88/filters:0
+batchnorm2d_96/beta:0
+batchnorm2d_96/gamma:0
+batchnorm2d_96/moving_mean:0
+batchnorm2d_96/moving_var:0
+conv2d_89/filters:0
+batchnorm2d_97/beta:0
+batchnorm2d_97/gamma:0
+batchnorm2d_97/moving_mean:0
+batchnorm2d_97/moving_var:0
+conv2d_90/filters:0
+batchnorm2d_98/beta:0
+batchnorm2d_98/gamma:0
+batchnorm2d_98/moving_mean:0
+batchnorm2d_98/moving_var:0
+conv2d_91/filters:0
+batchnorm2d_99/beta:0
+batchnorm2d_99/gamma:0
+batchnorm2d_99/moving_mean:0
+batchnorm2d_99/moving_var:0
+conv_route_3/filters:0
+batchnorm2d_100/beta:0
+batchnorm2d_100/gamma:0
+batchnorm2d_100/moving_mean:0
+batchnorm2d_100/moving_var:0
+conv_route_4/filters:0
+conv2d_92/filters:0
+conv2d_92/biases:0
+batchnorm2d_101/beta:0
+batchnorm2d_101/gamma:0
+batchnorm2d_101/moving_mean:0
+batchnorm2d_101/moving_var:0
+conv2d_93/filters:0
+batchnorm2d_102/beta:0
+batchnorm2d_102/gamma:0
+batchnorm2d_102/moving_mean:0
+batchnorm2d_102/moving_var:0
+conv2d_94/filters:0
+batchnorm2d_103/beta:0
+batchnorm2d_103/gamma:0
+batchnorm2d_103/moving_mean:0
+batchnorm2d_103/moving_var:0
+conv2d_95/filters:0
+batchnorm2d_104/beta:0
+batchnorm2d_104/gamma:0
+batchnorm2d_104/moving_mean:0
+batchnorm2d_104/moving_var:0
+conv2d_96/filters:0
+batchnorm2d_105/beta:0
+batchnorm2d_105/gamma:0
+batchnorm2d_105/moving_mean:0
+batchnorm2d_105/moving_var:0
+conv2d_97/filters:0
+batchnorm2d_106/beta:0
+batchnorm2d_106/gamma:0
+batchnorm2d_106/moving_mean:0
+batchnorm2d_106/moving_var:0
+conv2d_98/filters:0
+batchnorm2d_107/beta:0
+batchnorm2d_107/gamma:0
+batchnorm2d_107/moving_mean:0
+batchnorm2d_107/moving_var:0
+conv2d_99/filters:0
+conv2d_99/biases:0
\ No newline at end of file
diff --git a/examples/model_zoo/model/weights_3.txt b/examples/model_zoo/model/weights_3.txt
new file mode 100644
index 0000000..b9ff6e1
--- /dev/null
+++ b/examples/model_zoo/model/weights_3.txt
@@ -0,0 +1,541 @@
+conv2d_1/filters:0
+batchnorm2d_1/beta:0
+batchnorm2d_1/gamma:0
+batchnorm2d_1/moving_mean:0
+batchnorm2d_1/moving_var:0
+conv2d_2/filters:0
+batchnorm2d_2/beta:0
+batchnorm2d_2/gamma:0
+batchnorm2d_2/moving_mean:0
+batchnorm2d_2/moving_var:0
+conv_rote_block_1/filters:0
+batchnorm2d_3/beta:0
+batchnorm2d_3/gamma:0
+batchnorm2d_3/moving_mean:0
+batchnorm2d_3/moving_var:0
+conv2d_3/filters:0
+batchnorm2d_4/beta:0
+batchnorm2d_4/gamma:0
+batchnorm2d_4/moving_mean:0
+batchnorm2d_4/moving_var:0
+conv2d_4/filters:0
+batchnorm2d_5/beta:0
+batchnorm2d_5/gamma:0
+batchnorm2d_5/moving_mean:0
+batchnorm2d_5/moving_var:0
+conv2d_5/filters:0
+batchnorm2d_6/beta:0
+batchnorm2d_6/gamma:0
+batchnorm2d_6/moving_mean:0
+batchnorm2d_6/moving_var:0
+conv2d_6/filters:0
+batchnorm2d_7/beta:0
+batchnorm2d_7/gamma:0
+batchnorm2d_7/moving_mean:0
+batchnorm2d_7/moving_var:0
+conv2d_7/filters:0
+batchnorm2d_8/beta:0
+batchnorm2d_8/gamma:0
+batchnorm2d_8/moving_mean:0
+batchnorm2d_8/moving_var:0
+conv2d_8/filters:0
+batchnorm2d_9/beta:0
+batchnorm2d_9/gamma:0
+batchnorm2d_9/moving_mean:0
+batchnorm2d_9/moving_var:0
+conv_rote_block_2/filters:0
+batchnorm2d_10/beta:0
+batchnorm2d_10/gamma:0
+batchnorm2d_10/moving_mean:0
+batchnorm2d_10/moving_var:0
+conv2d_9/filters:0
+batchnorm2d_11/beta:0
+batchnorm2d_11/gamma:0
+batchnorm2d_11/moving_mean:0
+batchnorm2d_11/moving_var:0
+conv2d_10/filters:0
+batchnorm2d_12/beta:0
+batchnorm2d_12/gamma:0
+batchnorm2d_12/moving_mean:0
+batchnorm2d_12/moving_var:0
+conv2d_11/filters:0
+batchnorm2d_13/beta:0
+batchnorm2d_13/gamma:0
+batchnorm2d_13/moving_mean:0
+batchnorm2d_13/moving_var:0
+conv2d_12/filters:0
+batchnorm2d_14/beta:0
+batchnorm2d_14/gamma:0
+batchnorm2d_14/moving_mean:0
+batchnorm2d_14/moving_var:0
+conv2d_13/filters:0
+batchnorm2d_15/beta:0
+batchnorm2d_15/gamma:0
+batchnorm2d_15/moving_mean:0
+batchnorm2d_15/moving_var:0
+conv2d_14/filters:0
+batchnorm2d_16/beta:0
+batchnorm2d_16/gamma:0
+batchnorm2d_16/moving_mean:0
+batchnorm2d_16/moving_var:0
+conv2d_15/filters:0
+batchnorm2d_17/beta:0
+batchnorm2d_17/gamma:0
+batchnorm2d_17/moving_mean:0
+batchnorm2d_17/moving_var:0
+conv2d_16/filters:0
+batchnorm2d_18/beta:0
+batchnorm2d_18/gamma:0
+batchnorm2d_18/moving_mean:0
+batchnorm2d_18/moving_var:0
+conv_rote_block_3/filters:0
+batchnorm2d_19/beta:0
+batchnorm2d_19/gamma:0
+batchnorm2d_19/moving_mean:0
+batchnorm2d_19/moving_var:0
+conv2d_17/filters:0
+batchnorm2d_20/beta:0
+batchnorm2d_20/gamma:0
+batchnorm2d_20/moving_mean:0
+batchnorm2d_20/moving_var:0
+conv2d_18/filters:0
+batchnorm2d_21/beta:0
+batchnorm2d_21/gamma:0
+batchnorm2d_21/moving_mean:0
+batchnorm2d_21/moving_var:0
+conv2d_19/filters:0
+batchnorm2d_22/beta:0
+batchnorm2d_22/gamma:0
+batchnorm2d_22/moving_mean:0
+batchnorm2d_22/moving_var:0
+conv2d_20/filters:0
+batchnorm2d_23/beta:0
+batchnorm2d_23/gamma:0
+batchnorm2d_23/moving_mean:0
+batchnorm2d_23/moving_var:0
+conv2d_21/filters:0
+batchnorm2d_24/beta:0
+batchnorm2d_24/gamma:0
+batchnorm2d_24/moving_mean:0
+batchnorm2d_24/moving_var:0
+conv2d_22/filters:0
+batchnorm2d_25/beta:0
+batchnorm2d_25/gamma:0
+batchnorm2d_25/moving_mean:0
+batchnorm2d_25/moving_var:0
+conv2d_23/filters:0
+batchnorm2d_26/beta:0
+batchnorm2d_26/gamma:0
+batchnorm2d_26/moving_mean:0
+batchnorm2d_26/moving_var:0
+conv2d_24/filters:0
+batchnorm2d_27/beta:0
+batchnorm2d_27/gamma:0
+batchnorm2d_27/moving_mean:0
+batchnorm2d_27/moving_var:0
+conv2d_25/filters:0
+batchnorm2d_28/beta:0
+batchnorm2d_28/gamma:0
+batchnorm2d_28/moving_mean:0
+batchnorm2d_28/moving_var:0
+conv2d_26/filters:0
+batchnorm2d_29/beta:0
+batchnorm2d_29/gamma:0
+batchnorm2d_29/moving_mean:0
+batchnorm2d_29/moving_var:0
+conv2d_27/filters:0
+batchnorm2d_30/beta:0
+batchnorm2d_30/gamma:0
+batchnorm2d_30/moving_mean:0
+batchnorm2d_30/moving_var:0
+conv2d_28/filters:0
+batchnorm2d_31/beta:0
+batchnorm2d_31/gamma:0
+batchnorm2d_31/moving_mean:0
+batchnorm2d_31/moving_var:0
+conv2d_29/filters:0
+batchnorm2d_32/beta:0
+batchnorm2d_32/gamma:0
+batchnorm2d_32/moving_mean:0
+batchnorm2d_32/moving_var:0
+conv2d_30/filters:0
+batchnorm2d_33/beta:0
+batchnorm2d_33/gamma:0
+batchnorm2d_33/moving_mean:0
+batchnorm2d_33/moving_var:0
+conv2d_31/filters:0
+batchnorm2d_34/beta:0
+batchnorm2d_34/gamma:0
+batchnorm2d_34/moving_mean:0
+batchnorm2d_34/moving_var:0
+conv2d_32/filters:0
+batchnorm2d_35/beta:0
+batchnorm2d_35/gamma:0
+batchnorm2d_35/moving_mean:0
+batchnorm2d_35/moving_var:0
+conv2d_33/filters:0
+batchnorm2d_36/beta:0
+batchnorm2d_36/gamma:0
+batchnorm2d_36/moving_mean:0
+batchnorm2d_36/moving_var:0
+conv2d_34/filters:0
+batchnorm2d_37/beta:0
+batchnorm2d_37/gamma:0
+batchnorm2d_37/moving_mean:0
+batchnorm2d_37/moving_var:0
+conv2d_35/filters:0
+batchnorm2d_38/beta:0
+batchnorm2d_38/gamma:0
+batchnorm2d_38/moving_mean:0
+batchnorm2d_38/moving_var:0
+conv2d_36/filters:0
+batchnorm2d_39/beta:0
+batchnorm2d_39/gamma:0
+batchnorm2d_39/moving_mean:0
+batchnorm2d_39/moving_var:0
+conv_rote_block_4/filters:0
+batchnorm2d_40/beta:0
+batchnorm2d_40/gamma:0
+batchnorm2d_40/moving_mean:0
+batchnorm2d_40/moving_var:0
+conv2d_37/filters:0
+batchnorm2d_41/beta:0
+batchnorm2d_41/gamma:0
+batchnorm2d_41/moving_mean:0
+batchnorm2d_41/moving_var:0
+conv2d_38/filters:0
+batchnorm2d_42/beta:0
+batchnorm2d_42/gamma:0
+batchnorm2d_42/moving_mean:0
+batchnorm2d_42/moving_var:0
+conv2d_39/filters:0
+batchnorm2d_43/beta:0
+batchnorm2d_43/gamma:0
+batchnorm2d_43/moving_mean:0
+batchnorm2d_43/moving_var:0
+conv2d_40/filters:0
+batchnorm2d_44/beta:0
+batchnorm2d_44/gamma:0
+batchnorm2d_44/moving_mean:0
+batchnorm2d_44/moving_var:0
+conv2d_41/filters:0
+batchnorm2d_45/beta:0
+batchnorm2d_45/gamma:0
+batchnorm2d_45/moving_mean:0
+batchnorm2d_45/moving_var:0
+conv2d_42/filters:0
+batchnorm2d_46/beta:0
+batchnorm2d_46/gamma:0
+batchnorm2d_46/moving_mean:0
+batchnorm2d_46/moving_var:0
+conv2d_43/filters:0
+batchnorm2d_47/beta:0
+batchnorm2d_47/gamma:0
+batchnorm2d_47/moving_mean:0
+batchnorm2d_47/moving_var:0
+conv2d_44/filters:0
+batchnorm2d_48/beta:0
+batchnorm2d_48/gamma:0
+batchnorm2d_48/moving_mean:0
+batchnorm2d_48/moving_var:0
+conv2d_45/filters:0
+batchnorm2d_49/beta:0
+batchnorm2d_49/gamma:0
+batchnorm2d_49/moving_mean:0
+batchnorm2d_49/moving_var:0
+conv2d_46/filters:0
+batchnorm2d_50/beta:0
+batchnorm2d_50/gamma:0
+batchnorm2d_50/moving_mean:0
+batchnorm2d_50/moving_var:0
+conv2d_47/filters:0
+batchnorm2d_51/beta:0
+batchnorm2d_51/gamma:0
+batchnorm2d_51/moving_mean:0
+batchnorm2d_51/moving_var:0
+conv2d_48/filters:0
+batchnorm2d_52/beta:0
+batchnorm2d_52/gamma:0
+batchnorm2d_52/moving_mean:0
+batchnorm2d_52/moving_var:0
+conv2d_49/filters:0
+batchnorm2d_53/beta:0
+batchnorm2d_53/gamma:0
+batchnorm2d_53/moving_mean:0
+batchnorm2d_53/moving_var:0
+conv2d_50/filters:0
+batchnorm2d_54/beta:0
+batchnorm2d_54/gamma:0
+batchnorm2d_54/moving_mean:0
+batchnorm2d_54/moving_var:0
+conv2d_51/filters:0
+batchnorm2d_55/beta:0
+batchnorm2d_55/gamma:0
+batchnorm2d_55/moving_mean:0
+batchnorm2d_55/moving_var:0
+conv2d_52/filters:0
+batchnorm2d_56/beta:0
+batchnorm2d_56/gamma:0
+batchnorm2d_56/moving_mean:0
+batchnorm2d_56/moving_var:0
+conv2d_53/filters:0
+batchnorm2d_57/beta:0
+batchnorm2d_57/gamma:0
+batchnorm2d_57/moving_mean:0
+batchnorm2d_57/moving_var:0
+conv2d_54/filters:0
+batchnorm2d_58/beta:0
+batchnorm2d_58/gamma:0
+batchnorm2d_58/moving_mean:0
+batchnorm2d_58/moving_var:0
+conv2d_55/filters:0
+batchnorm2d_59/beta:0
+batchnorm2d_59/gamma:0
+batchnorm2d_59/moving_mean:0
+batchnorm2d_59/moving_var:0
+conv2d_56/filters:0
+batchnorm2d_60/beta:0
+batchnorm2d_60/gamma:0
+batchnorm2d_60/moving_mean:0
+batchnorm2d_60/moving_var:0
+conv_rote_block_5/filters:0
+batchnorm2d_61/beta:0
+batchnorm2d_61/gamma:0
+batchnorm2d_61/moving_mean:0
+batchnorm2d_61/moving_var:0
+conv2d_57/filters:0
+batchnorm2d_62/beta:0
+batchnorm2d_62/gamma:0
+batchnorm2d_62/moving_mean:0
+batchnorm2d_62/moving_var:0
+conv2d_58/filters:0
+batchnorm2d_63/beta:0
+batchnorm2d_63/gamma:0
+batchnorm2d_63/moving_mean:0
+batchnorm2d_63/moving_var:0
+conv2d_59/filters:0
+batchnorm2d_64/beta:0
+batchnorm2d_64/gamma:0
+batchnorm2d_64/moving_mean:0
+batchnorm2d_64/moving_var:0
+conv2d_60/filters:0
+batchnorm2d_65/beta:0
+batchnorm2d_65/gamma:0
+batchnorm2d_65/moving_mean:0
+batchnorm2d_65/moving_var:0
+conv2d_61/filters:0
+batchnorm2d_66/beta:0
+batchnorm2d_66/gamma:0
+batchnorm2d_66/moving_mean:0
+batchnorm2d_66/moving_var:0
+conv2d_62/filters:0
+batchnorm2d_67/beta:0
+batchnorm2d_67/gamma:0
+batchnorm2d_67/moving_mean:0
+batchnorm2d_67/moving_var:0
+conv2d_63/filters:0
+batchnorm2d_68/beta:0
+batchnorm2d_68/gamma:0
+batchnorm2d_68/moving_mean:0
+batchnorm2d_68/moving_var:0
+conv2d_64/filters:0
+batchnorm2d_69/beta:0
+batchnorm2d_69/gamma:0
+batchnorm2d_69/moving_mean:0
+batchnorm2d_69/moving_var:0
+conv2d_65/filters:0
+batchnorm2d_70/beta:0
+batchnorm2d_70/gamma:0
+batchnorm2d_70/moving_mean:0
+batchnorm2d_70/moving_var:0
+conv2d_66/filters:0
+batchnorm2d_71/beta:0
+batchnorm2d_71/gamma:0
+batchnorm2d_71/moving_mean:0
+batchnorm2d_71/moving_var:0
+conv2d_67/filters:0
+batchnorm2d_72/beta:0
+batchnorm2d_72/gamma:0
+batchnorm2d_72/moving_mean:0
+batchnorm2d_72/moving_var:0
+conv2d_68/filters:0
+batchnorm2d_73/beta:0
+batchnorm2d_73/gamma:0
+batchnorm2d_73/moving_mean:0
+batchnorm2d_73/moving_var:0
+conv2d_69/filters:0
+batchnorm2d_74/beta:0
+batchnorm2d_74/gamma:0
+batchnorm2d_74/moving_mean:0
+batchnorm2d_74/moving_var:0
+conv2d_70/filters:0
+batchnorm2d_75/beta:0
+batchnorm2d_75/gamma:0
+batchnorm2d_75/moving_mean:0
+batchnorm2d_75/moving_var:0
+conv2d_71/filters:0
+batchnorm2d_76/beta:0
+batchnorm2d_76/gamma:0
+batchnorm2d_76/moving_mean:0
+batchnorm2d_76/moving_var:0
+conv2d_72/filters:0
+batchnorm2d_77/beta:0
+batchnorm2d_77/gamma:0
+batchnorm2d_77/moving_mean:0
+batchnorm2d_77/moving_var:0
+conv2d_73/filters:0
+batchnorm2d_78/beta:0
+batchnorm2d_78/gamma:0
+batchnorm2d_78/moving_mean:0
+batchnorm2d_78/moving_var:0
+conv2d_74/filters:0
+batchnorm2d_79/beta:0
+batchnorm2d_79/gamma:0
+batchnorm2d_79/moving_mean:0
+batchnorm2d_79/moving_var:0
+conv_yolo_1/filters:0
+batchnorm2d_80/beta:0
+batchnorm2d_80/gamma:0
+batchnorm2d_80/moving_mean:0
+batchnorm2d_80/moving_var:0
+conv2d_75/filters:0
+batchnorm2d_81/beta:0
+batchnorm2d_81/gamma:0
+batchnorm2d_81/moving_mean:0
+batchnorm2d_81/moving_var:0
+conv2d_76/filters:0
+batchnorm2d_82/beta:0
+batchnorm2d_82/gamma:0
+batchnorm2d_82/moving_mean:0
+batchnorm2d_82/moving_var:0
+conv2d_77/filters:0
+batchnorm2d_83/beta:0
+batchnorm2d_83/gamma:0
+batchnorm2d_83/moving_mean:0
+batchnorm2d_83/moving_var:0
+conv2d_78/filters:0
+batchnorm2d_84/beta:0
+batchnorm2d_84/gamma:0
+batchnorm2d_84/moving_mean:0
+batchnorm2d_84/moving_var:0
+conv2d_79/filters:0
+batchnorm2d_85/beta:0
+batchnorm2d_85/gamma:0
+batchnorm2d_85/moving_mean:0
+batchnorm2d_85/moving_var:0
+conv2d_80/filters:0
+batchnorm2d_86/beta:0
+batchnorm2d_86/gamma:0
+batchnorm2d_86/moving_mean:0
+batchnorm2d_86/moving_var:0
+conv_yolo_2/filters:0
+batchnorm2d_87/beta:0
+batchnorm2d_87/gamma:0
+batchnorm2d_87/moving_mean:0
+batchnorm2d_87/moving_var:0
+conv2d_81/filters:0
+batchnorm2d_88/beta:0
+batchnorm2d_88/gamma:0
+batchnorm2d_88/moving_mean:0
+batchnorm2d_88/moving_var:0
+conv2d_82/filters:0
+batchnorm2d_89/beta:0
+batchnorm2d_89/gamma:0
+batchnorm2d_89/moving_mean:0
+batchnorm2d_89/moving_var:0
+conv2d_83/filters:0
+batchnorm2d_90/beta:0
+batchnorm2d_90/gamma:0
+batchnorm2d_90/moving_mean:0
+batchnorm2d_90/moving_var:0
+conv2d_84/filters:0
+batchnorm2d_91/beta:0
+batchnorm2d_91/gamma:0
+batchnorm2d_91/moving_mean:0
+batchnorm2d_91/moving_var:0
+conv2d_85/filters:0
+batchnorm2d_92/beta:0
+batchnorm2d_92/gamma:0
+batchnorm2d_92/moving_mean:0
+batchnorm2d_92/moving_var:0
+conv_route_1/filters:0
+batchnorm2d_93/beta:0
+batchnorm2d_93/gamma:0
+batchnorm2d_93/moving_mean:0
+batchnorm2d_93/moving_var:0
+conv2d_86/filters:0
+conv2d_86/biases:0
+conv_route_2/filters:0
+batchnorm2d_94/beta:0
+batchnorm2d_94/gamma:0
+batchnorm2d_94/moving_mean:0
+batchnorm2d_94/moving_var:0
+conv2d_87/filters:0
+batchnorm2d_95/beta:0
+batchnorm2d_95/gamma:0
+batchnorm2d_95/moving_mean:0
+batchnorm2d_95/moving_var:0
+conv2d_88/filters:0
+batchnorm2d_96/beta:0
+batchnorm2d_96/gamma:0
+batchnorm2d_96/moving_mean:0
+batchnorm2d_96/moving_var:0
+conv2d_89/filters:0
+batchnorm2d_97/beta:0
+batchnorm2d_97/gamma:0
+batchnorm2d_97/moving_mean:0
+batchnorm2d_97/moving_var:0
+conv2d_90/filters:0
+batchnorm2d_98/beta:0
+batchnorm2d_98/gamma:0
+batchnorm2d_98/moving_mean:0
+batchnorm2d_98/moving_var:0
+conv2d_91/filters:0
+batchnorm2d_99/beta:0
+batchnorm2d_99/gamma:0
+batchnorm2d_99/moving_mean:0
+batchnorm2d_99/moving_var:0
+conv_route_3/filters:0
+batchnorm2d_100/beta:0
+batchnorm2d_100/gamma:0
+batchnorm2d_100/moving_mean:0
+batchnorm2d_100/moving_var:0
+conv2d_92/filters:0
+conv2d_92/biases:0
+conv_route_4/filters:0
+batchnorm2d_101/beta:0
+batchnorm2d_101/gamma:0
+batchnorm2d_101/moving_mean:0
+batchnorm2d_101/moving_var:0
+conv2d_93/filters:0
+batchnorm2d_102/beta:0
+batchnorm2d_102/gamma:0
+batchnorm2d_102/moving_mean:0
+batchnorm2d_102/moving_var:0
+conv2d_94/filters:0
+batchnorm2d_103/beta:0
+batchnorm2d_103/gamma:0
+batchnorm2d_103/moving_mean:0
+batchnorm2d_103/moving_var:0
+conv2d_95/filters:0
+batchnorm2d_104/beta:0
+batchnorm2d_104/gamma:0
+batchnorm2d_104/moving_mean:0
+batchnorm2d_104/moving_var:0
+conv2d_96/filters:0
+batchnorm2d_105/beta:0
+batchnorm2d_105/gamma:0
+batchnorm2d_105/moving_mean:0
+batchnorm2d_105/moving_var:0
+conv2d_97/filters:0
+batchnorm2d_106/beta:0
+batchnorm2d_106/gamma:0
+batchnorm2d_106/moving_mean:0
+batchnorm2d_106/moving_var:0
+conv2d_98/filters:0
+batchnorm2d_107/beta:0
+batchnorm2d_107/gamma:0
+batchnorm2d_107/moving_mean:0
+batchnorm2d_107/moving_var:0
+conv2d_99/filters:0
+conv2d_99/biases:0
\ No newline at end of file
diff --git a/examples/model_zoo/model/yolov4_weights3_config.txt b/examples/model_zoo/model/yolov4_weights3_config.txt
new file mode 100644
index 0000000..5f31bb5
--- /dev/null
+++ b/examples/model_zoo/model/yolov4_weights3_config.txt
@@ -0,0 +1,541 @@
+layer_with_weights-0/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-1/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-1/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-1/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-1/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-2/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-3/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-3/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-3/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-3/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-11/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-13/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-13/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-13/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-13/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-4/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-5/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-5/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-5/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-5/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-6/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-7/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-7/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-7/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-7/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-8/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-9/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-9/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-9/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-9/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-10/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-12/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-12/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-12/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-12/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-14/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-15/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-15/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-15/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-15/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-16/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-17/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-17/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-17/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-17/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-29/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-31/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-31/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-31/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-31/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-18/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-19/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-19/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-19/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-19/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-20/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-21/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-21/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-21/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-21/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-22/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-23/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-23/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-23/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-23/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-24/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-25/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-25/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-25/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-25/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-26/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-27/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-27/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-27/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-27/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-28/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-30/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-30/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-30/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-30/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-32/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-33/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-33/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-33/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-33/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-34/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-35/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-35/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-35/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-35/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-71/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-73/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-73/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-73/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-73/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-36/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-37/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-37/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-37/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-37/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-38/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-39/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-39/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-39/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-39/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-40/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-41/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-41/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-41/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-41/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-42/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-43/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-43/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-43/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-43/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-44/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-45/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-45/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-45/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-45/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-46/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-47/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-47/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-47/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-47/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-48/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-49/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-49/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-49/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-49/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-50/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-51/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-51/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-51/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-51/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-52/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-53/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-53/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-53/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-53/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-54/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-55/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-55/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-55/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-55/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-56/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-57/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-57/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-57/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-57/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-58/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-59/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-59/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-59/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-59/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-60/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-61/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-61/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-61/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-61/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-62/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-63/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-63/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-63/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-63/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-64/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-65/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-65/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-65/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-65/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-66/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-67/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-67/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-67/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-67/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-68/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-69/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-69/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-69/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-69/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-70/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-72/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-72/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-72/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-72/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-74/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-75/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-75/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-75/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-75/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-76/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-77/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-77/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-77/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-77/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-113/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-115/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-115/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-115/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-115/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-78/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-79/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-79/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-79/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-79/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-80/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-81/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-81/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-81/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-81/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-82/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-83/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-83/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-83/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-83/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-84/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-85/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-85/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-85/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-85/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-86/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-87/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-87/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-87/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-87/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-88/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-89/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-89/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-89/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-89/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-90/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-91/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-91/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-91/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-91/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-92/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-93/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-93/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-93/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-93/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-94/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-95/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-95/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-95/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-95/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-96/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-97/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-97/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-97/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-97/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-98/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-99/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-99/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-99/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-99/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-100/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-101/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-101/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-101/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-101/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-102/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-103/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-103/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-103/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-103/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-104/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-105/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-105/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-105/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-105/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-106/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-107/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-107/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-107/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-107/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-108/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-109/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-109/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-109/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-109/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-110/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-111/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-111/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-111/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-111/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-112/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-114/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-114/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-114/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-114/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-116/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-117/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-117/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-117/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-117/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-118/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-119/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-119/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-119/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-119/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-139/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-141/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-141/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-141/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-141/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-120/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-121/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-121/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-121/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-121/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-122/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-123/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-123/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-123/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-123/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-124/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-125/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-125/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-125/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-125/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-126/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-127/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-127/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-127/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-127/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-128/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-129/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-129/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-129/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-129/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-130/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-131/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-131/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-131/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-131/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-132/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-133/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-133/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-133/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-133/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-134/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-135/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-135/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-135/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-135/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-136/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-137/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-137/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-137/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-137/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-138/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-140/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-140/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-140/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-140/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-142/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-143/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-143/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-143/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-143/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-144/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-145/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-145/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-145/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-145/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-146/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-147/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-147/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-147/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-147/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-148/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-149/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-149/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-149/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-149/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-150/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-151/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-151/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-151/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-151/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-152/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-153/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-153/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-153/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-153/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-154/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-155/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-155/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-155/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-155/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-156/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-158/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-158/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-158/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-158/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-157/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-159/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-159/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-159/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-159/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-160/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-161/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-161/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-161/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-161/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-162/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-163/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-163/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-163/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-163/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-164/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-165/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-165/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-165/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-165/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-166/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-167/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-167/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-167/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-167/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-168/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-169/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-169/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-169/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-169/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-170/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-172/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-172/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-172/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-172/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-171/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-173/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-173/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-173/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-173/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-174/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-175/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-175/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-175/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-175/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-176/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-177/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-177/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-177/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-177/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-178/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-179/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-179/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-179/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-179/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-180/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-181/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-181/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-181/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-181/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-182/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-183/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-183/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-183/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-183/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-208/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-211/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-211/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-211/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-211/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-214/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-214/bias/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-184/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-185/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-185/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-185/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-185/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-186/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-187/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-187/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-187/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-187/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-188/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-189/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-189/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-189/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-189/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-190/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-191/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-191/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-191/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-191/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-192/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-193/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-193/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-193/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-193/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-194/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-195/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-195/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-195/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-195/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-209/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-212/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-212/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-212/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-212/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-215/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-215/bias/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-196/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-197/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-197/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-197/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-197/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-198/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-199/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-199/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-199/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-199/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-200/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-201/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-201/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-201/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-201/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-202/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-203/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-203/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-203/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-203/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-204/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-205/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-205/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-205/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-205/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-206/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-207/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-207/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-207/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-207/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-210/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-213/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-213/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-213/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-213/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-216/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-216/bias/.ATTRIBUTES/VARIABLE_VALUE
\ No newline at end of file
diff --git a/examples/model_zoo/model/yolov4_weights_config.txt b/examples/model_zoo/model/yolov4_weights_config.txt
new file mode 100644
index 0000000..2c28be0
--- /dev/null
+++ b/examples/model_zoo/model/yolov4_weights_config.txt
@@ -0,0 +1,541 @@
+layer_with_weights-0/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-1/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-1/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-1/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-1/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-2/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-3/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-3/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-3/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-3/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-11/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-4/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-13/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-13/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-13/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-13/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-5/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-5/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-5/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-5/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-6/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-7/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-7/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-7/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-7/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-8/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-9/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-9/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-9/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-9/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-10/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-12/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-12/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-12/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-12/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-14/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-15/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-15/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-15/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-15/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-16/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-17/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-17/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-17/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-17/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-29/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-18/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-31/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-31/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-31/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-31/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-19/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-19/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-19/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-19/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-20/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-21/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-21/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-21/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-21/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-22/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-23/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-23/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-23/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-23/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-24/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-25/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-25/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-25/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-25/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-26/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-27/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-27/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-27/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-27/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-28/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-30/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-30/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-30/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-30/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-32/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-33/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-33/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-33/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-33/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-34/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-35/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-35/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-35/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-35/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-71/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-36/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-73/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-73/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-73/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-73/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-37/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-37/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-37/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-37/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-38/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-39/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-39/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-39/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-39/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-40/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-41/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-41/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-41/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-41/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-42/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-43/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-43/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-43/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-43/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-44/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-45/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-45/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-45/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-45/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-46/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-47/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-47/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-47/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-47/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-48/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-49/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-49/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-49/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-49/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-50/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-51/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-51/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-51/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-51/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-52/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-53/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-53/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-53/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-53/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-54/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-55/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-55/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-55/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-55/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-56/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-57/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-57/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-57/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-57/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-58/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-59/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-59/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-59/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-59/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-60/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-61/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-61/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-61/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-61/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-62/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-63/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-63/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-63/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-63/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-64/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-65/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-65/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-65/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-65/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-66/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-67/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-67/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-67/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-67/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-68/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-69/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-69/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-69/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-69/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-70/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-72/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-72/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-72/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-72/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-74/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-75/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-75/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-75/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-75/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-171/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-173/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-173/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-173/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-173/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-76/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-77/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-77/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-77/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-77/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-113/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-78/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-115/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-115/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-115/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-115/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-79/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-79/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-79/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-79/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-80/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-81/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-81/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-81/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-81/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-82/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-83/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-83/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-83/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-83/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-84/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-85/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-85/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-85/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-85/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-86/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-87/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-87/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-87/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-87/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-88/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-89/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-89/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-89/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-89/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-90/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-91/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-91/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-91/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-91/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-92/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-93/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-93/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-93/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-93/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-94/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-95/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-95/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-95/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-95/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-96/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-97/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-97/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-97/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-97/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-98/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-99/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-99/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-99/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-99/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-100/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-101/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-101/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-101/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-101/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-102/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-103/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-103/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-103/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-103/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-104/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-105/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-105/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-105/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-105/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-106/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-107/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-107/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-107/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-107/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-108/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-109/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-109/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-109/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-109/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-110/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-111/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-111/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-111/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-111/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-112/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-114/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-114/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-114/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-114/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-116/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-117/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-117/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-117/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-117/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-157/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-159/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-159/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-159/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-159/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-118/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-119/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-119/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-119/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-119/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-139/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-120/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-141/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-141/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-141/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-141/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-121/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-121/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-121/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-121/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-122/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-123/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-123/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-123/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-123/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-124/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-125/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-125/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-125/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-125/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-126/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-127/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-127/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-127/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-127/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-128/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-129/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-129/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-129/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-129/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-130/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-131/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-131/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-131/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-131/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-132/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-133/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-133/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-133/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-133/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-134/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-135/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-135/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-135/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-135/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-136/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-137/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-137/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-137/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-137/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-138/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-140/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-140/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-140/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-140/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-142/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-143/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-143/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-143/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-143/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-144/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-145/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-145/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-145/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-145/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-146/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-147/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-147/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-147/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-147/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-148/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-149/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-149/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-149/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-149/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-150/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-151/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-151/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-151/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-151/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-152/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-153/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-153/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-153/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-153/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-154/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-155/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-155/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-155/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-155/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-156/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-158/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-158/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-158/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-158/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-160/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-161/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-161/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-161/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-161/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-162/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-163/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-163/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-163/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-163/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-164/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-165/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-165/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-165/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-165/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-166/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-167/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-167/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-167/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-167/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-168/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-169/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-169/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-169/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-169/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-170/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-172/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-172/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-172/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-172/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-174/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-175/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-175/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-175/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-175/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-176/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-177/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-177/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-177/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-177/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-178/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-179/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-179/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-179/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-179/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-180/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-181/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-181/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-181/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-181/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-182/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-183/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-183/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-183/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-183/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-208/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-211/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-211/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-211/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-211/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-184/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-214/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-214/bias/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-185/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-185/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-185/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-185/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-186/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-187/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-187/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-187/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-187/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-188/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-189/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-189/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-189/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-189/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-190/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-191/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-191/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-191/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-191/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-192/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-193/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-193/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-193/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-193/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-194/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-195/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-195/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-195/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-195/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-209/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-212/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-212/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-212/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-212/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-196/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-215/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-215/bias/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-197/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-197/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-197/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-197/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-198/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-199/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-199/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-199/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-199/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-200/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-201/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-201/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-201/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-201/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-202/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-203/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-203/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-203/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-203/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-204/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-205/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-205/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-205/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-205/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-206/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-207/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-207/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-207/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-207/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-210/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-213/beta/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-213/gamma/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-213/moving_mean/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-213/moving_variance/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-216/kernel/.ATTRIBUTES/VARIABLE_VALUE
+layer_with_weights-216/bias/.ATTRIBUTES/VARIABLE_VALUE
\ No newline at end of file
diff --git a/examples/model_zoo/pretrained_resnet50.py b/examples/model_zoo/pretrained_resnet50.py
new file mode 100644
index 0000000..cac33eb
--- /dev/null
+++ b/examples/model_zoo/pretrained_resnet50.py
@@ -0,0 +1,32 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+ResNet50 for ImageNet using TL models
+
+"""
+
+import time
+import numpy as np
+import tensorlayer as tl
+from examples.model_zoo.imagenet_classes import class_names
+from examples.model_zoo.resnet import ResNet50
+
+tl.logging.set_verbosity(tl.logging.DEBUG)
+
+# get the whole model
+resnet = ResNet50(pretrained=False)
+resnet.set_eval()
+
+img1 = tl.vis.read_image('data/tiger.jpeg')
+img1 = tl.prepro.imresize(img1, (224, 224))[:, :, ::-1]
+img1 = img1 - np.array([103.939, 116.779, 123.68]).reshape((1, 1, 3))
+
+img1 = img1.astype(np.float32)[np.newaxis, ...]
+
+start_time = time.time()
+output = resnet(img1)
+prob = tl.ops.softmax(output)[0].numpy()
+print(" End time : %.5ss" % (time.time() - start_time))
+preds = (np.argsort(prob)[::-1])[0:5]
+for p in preds:
+ print(class_names[p], prob[p])
diff --git a/examples/model_zoo/pretrained_vgg16.py b/examples/model_zoo/pretrained_vgg16.py
new file mode 100644
index 0000000..9bf4264
--- /dev/null
+++ b/examples/model_zoo/pretrained_vgg16.py
@@ -0,0 +1,29 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""VGG-16 for ImageNet using TL models."""
+
+import time
+
+import numpy as np
+import tensorflow as tf
+
+import tensorlayer as tl
+from examples.model_zoo.imagenet_classes import class_names
+from examples.model_zoo.vgg import vgg16
+
+tl.logging.set_verbosity(tl.logging.DEBUG)
+
+# get the whole model
+vgg = vgg16(pretrained=True)
+vgg.set_eval()
+
+img = tl.vis.read_image('data/tiger.jpeg')
+img = tl.prepro.imresize(img, (224, 224)).astype(np.float32) / 255
+
+start_time = time.time()
+output = vgg(img)
+probs = tf.nn.softmax(output)[0].numpy()
+print(" End time : %.5ss" % (time.time() - start_time))
+preds = (np.argsort(probs)[::-1])[0:5]
+for p in preds:
+ print(class_names[p], probs[p])
diff --git a/examples/model_zoo/pretrained_yolov4.py b/examples/model_zoo/pretrained_yolov4.py
new file mode 100644
index 0000000..c8d3908
--- /dev/null
+++ b/examples/model_zoo/pretrained_yolov4.py
@@ -0,0 +1,28 @@
+import numpy as np
+import cv2
+from PIL import Image
+from examples.model_zoo.common import yolo4_input_processing, yolo4_output_processing, \
+ result_to_json, read_class_names, draw_boxes_and_labels_to_image_with_json
+from examples.model_zoo.yolo import YOLOv4
+import tensorlayer as tl
+
+tl.logging.set_verbosity(tl.logging.DEBUG)
+
+INPUT_SIZE = 416
+image_path = './data/kite.jpg'
+
+class_names = read_class_names('./model/coco.names')
+original_image = cv2.imread(image_path)
+image = cv2.cvtColor(np.array(original_image), cv2.COLOR_BGR2RGB)
+
+model = YOLOv4(NUM_CLASS=80, pretrained=True)
+model.set_eval()
+
+batch_data = yolo4_input_processing(original_image)
+feature_maps = model(batch_data)
+pred_bbox = yolo4_output_processing(feature_maps)
+json_result = result_to_json(image, pred_bbox)
+
+image = draw_boxes_and_labels_to_image_with_json(image, json_result, class_names)
+image = Image.fromarray(image.astype(np.uint8))
+image.show()
\ No newline at end of file
diff --git a/examples/model_zoo/resnet.py b/examples/model_zoo/resnet.py
new file mode 100644
index 0000000..c57bef9
--- /dev/null
+++ b/examples/model_zoo/resnet.py
@@ -0,0 +1,225 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""ResNet for ImageNet.
+
+# Reference:
+- [Deep Residual Learning for Image Recognition](
+ https://arxiv.org/abs/1512.03385) (CVPR 2016 Best Paper Award)
+
+"""
+
+import os
+
+import tensorlayer as tl
+
+from tensorlayer import logging
+from tensorlayer.files import (assign_weights, maybe_download_and_extract)
+from tensorlayer.layers import (BatchNorm, Conv2d, Dense, Elementwise, GlobalMeanPool2d, Input, MaxPool2d)
+from tensorlayer.layers import Module, SequentialLayer
+
+__all__ = [
+ 'ResNet50',
+]
+
+block_names = ['2a', '2b', '2c', '3a', '3b', '3c', '3d', '4a', '4b', '4c', '4d', '4e', '4f', '5a', '5b', '5c'
+ ] + ['avg_pool', 'fc1000']
+block_filters = [[64, 64, 256], [128, 128, 512], [256, 256, 1024], [512, 512, 2048]]
+in_channels_conv = [64, 256, 512, 1024]
+in_channels_identity = [256, 512, 1024, 2048]
+henorm = tl.initializers.he_normal()
+
+class identity_block(Module):
+ """The identity block where there is no conv layer at shortcut.
+
+ Parameters
+ ----------
+ input : tf tensor
+ Input tensor from above layer.
+ kernel_size : int
+ The kernel size of middle conv layer at main path.
+ n_filters : list of integers
+ The numbers of filters for 3 conv layer at main path.
+ stage : int
+ Current stage label.
+ block : str
+ Current block label.
+
+ Returns
+ -------
+ Output tensor of this block.
+
+ """
+ def __init__(self, kernel_size, n_filters, stage, block):
+ super(identity_block, self).__init__()
+ filters1, filters2, filters3 = n_filters
+ _in_channels = in_channels_identity[stage-2]
+ conv_name_base = 'res' + str(stage) + block + '_branch'
+ bn_name_base = 'bn' + str(stage) + block + '_branch'
+
+ self.conv1 = Conv2d(filters1, (1, 1), W_init=henorm, name=conv_name_base + '2a', in_channels=_in_channels)
+ self.bn1 = BatchNorm(name=bn_name_base + '2a', act='relu', num_features=filters1)
+
+ ks = (kernel_size, kernel_size)
+ self.conv2 = Conv2d(filters2, ks, padding='SAME', W_init=henorm, name=conv_name_base + '2b', in_channels=filters1)
+ self.bn2 = BatchNorm(name=bn_name_base + '2b', act='relu', num_features=filters2)
+
+ self.conv3 = Conv2d(filters3, (1, 1), W_init=henorm, name=conv_name_base + '2c', in_channels=filters2)
+ self.bn3 = BatchNorm(name=bn_name_base + '2c', num_features=filters3)
+
+ self.add = Elementwise(tl.add, act='relu')
+
+ def forward(self, inputs):
+ output = self.conv1(inputs)
+ output = self.bn1(output)
+ output = self.conv2(output)
+ output = self.bn2(output)
+ output = self.conv3(output)
+ output = self.bn3(output)
+ result = self.add([output, inputs])
+ return result
+
+
+class conv_block(Module):
+ def __init__(self, kernel_size, n_filters, stage, block, strides=(2, 2)):
+ super(conv_block, self).__init__()
+ filters1, filters2, filters3 = n_filters
+ _in_channels = in_channels_conv[stage-2]
+ conv_name_base = 'res' + str(stage) + block + '_branch'
+ bn_name_base = 'bn' + str(stage) + block + '_branch'
+ self.conv1 = Conv2d(filters1, (1, 1), strides=strides, W_init=henorm, name=conv_name_base + '2a', in_channels=_in_channels)
+ self.bn1 = BatchNorm(name=bn_name_base + '2a', act='relu', num_features=filters1)
+
+ ks = (kernel_size, kernel_size)
+ self.conv2 = Conv2d(filters2, ks, padding='SAME', W_init=henorm, name=conv_name_base + '2b', in_channels=filters1)
+ self.bn2 = BatchNorm(name=bn_name_base + '2b', act='relu', num_features=filters2)
+
+ self.conv3 = Conv2d(filters3, (1, 1), W_init=henorm, name=conv_name_base + '2c', in_channels=filters2)
+ self.bn3 = BatchNorm(name=bn_name_base + '2c', num_features=filters3)
+
+ self.shortcut_conv = Conv2d(filters3, (1, 1), strides=strides, W_init=henorm, name=conv_name_base + '1', in_channels=_in_channels)
+ self.shortcut_bn = BatchNorm(name=bn_name_base + '1', num_features=filters3)
+
+ self.add = Elementwise(tl.add, act='relu')
+
+ def forward(self, inputs):
+ output = self.conv1(inputs)
+ output = self.bn1(output)
+ output = self.conv2(output)
+ output = self.bn2(output)
+ output = self.conv3(output)
+ output = self.bn3(output)
+
+ shortcut = self.shortcut_conv(inputs)
+ shortcut = self.shortcut_bn(shortcut)
+
+ result = self.add([output, shortcut])
+ return result
+
+
+class ResNet50_model(Module):
+ def __init__(self, end_with='fc1000', n_classes=1000):
+ super(ResNet50_model, self).__init__()
+ self.end_with = end_with
+ self.n_classes = n_classes
+ self.conv1 = Conv2d(64, (7, 7), in_channels=3, strides=(2, 2), padding='SAME', W_init=henorm, name='conv1')
+ self.bn_conv1 = BatchNorm(name='bn_conv1', act="relu", num_features=64)
+ self.max_pool1 = MaxPool2d((3, 3), strides=(2, 2), name='max_pool1')
+ self.res_layer = self.make_layer()
+
+ def forward(self, inputs):
+ z = self.conv1(inputs)
+ z = self.bn_conv1(z)
+ z = self.max_pool1(z)
+ z = self.res_layer(z)
+ return z
+
+ def make_layer(self):
+ layer_list = []
+ for i, block_name in enumerate(block_names):
+ if len(block_name) == 2:
+ stage = int(block_name[0])
+ block = block_name[1]
+ if block == 'a':
+ strides = (1, 1) if stage == 2 else (2, 2)
+ layer_list.append(conv_block(3, block_filters[stage - 2], stage=stage, block=block, strides=strides))
+ else:
+ layer_list.append(identity_block(3, block_filters[stage - 2], stage=stage, block=block))
+ elif block_name == 'avg_pool':
+ layer_list.append(GlobalMeanPool2d(name='avg_pool'))
+ elif block_name == 'fc1000':
+ layer_list.append(Dense(self.n_classes, name='fc1000', in_channels=2048))
+
+ if block_name == self.end_with:
+ break
+ return SequentialLayer(layer_list)
+
+
+def ResNet50(pretrained=False, end_with='fc1000', n_classes=1000):
+ """Pre-trained MobileNetV1 model (static mode). Input shape [?, 224, 224, 3].
+ To use pretrained model, input should be in BGR format and subtracted from ImageNet mean [103.939, 116.779, 123.68].
+
+ Parameters
+ ----------
+ pretrained : boolean
+ Whether to load pretrained weights. Default False.
+ end_with : str
+ The end point of the model [conv, depth1, depth2 ... depth13, globalmeanpool, out].
+ Default ``out`` i.e. the whole model.
+ n_classes : int
+ Number of classes in final prediction.
+ name : None or str
+ Name for this model.
+
+ Examples
+ ---------
+ Classify ImageNet classes, see `tutorial_models_resnet50.py`
+ TODO Modify the usage example according to the model storage location
+ >>> # get the whole model with pretrained weights
+ >>> resnet = tl.models.ResNet50(pretrained=True)
+ >>> # use for inferencing
+ >>> output = resnet(img1, is_train=False)
+ >>> prob = tf.nn.softmax(output)[0].numpy()
+
+ Extract the features before fc layer
+ >>> resnet = tl.models.ResNet50(pretrained=True, end_with='5c')
+ >>> output = resnet(img1, is_train=False)
+
+ Returns
+ -------
+ ResNet50 model.
+
+ """
+
+ network = ResNet50_model(end_with=end_with, n_classes=n_classes)
+
+ if pretrained:
+ restore_params(network)
+
+ return network
+
+
+def restore_params(network, path='models'):
+ logging.info("Restore pre-trained parameters")
+ maybe_download_and_extract(
+ 'resnet50_weights_tf_dim_ordering_tf_kernels.h5',
+ path,
+ 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/',
+ ) # ls -al
+ try:
+ import h5py
+ except Exception:
+ raise ImportError('h5py not imported')
+
+ f = h5py.File(os.path.join(path, 'resnet50_weights_tf_dim_ordering_tf_kernels.h5'), 'r')
+
+ for layer in network.all_layers:
+ if len(layer.all_weights) == 0:
+ continue
+ w_names = list(f[layer.name])
+ params = [f[layer.name][n][:] for n in w_names]
+ # if 'bn' in layer.name:
+ # params = [x.reshape(1, 1, 1, -1) for x in params]
+ assign_weights(params, layer)
+ del params
+
+ f.close()
diff --git a/examples/model_zoo/vgg.py b/examples/model_zoo/vgg.py
new file mode 100644
index 0000000..779635d
--- /dev/null
+++ b/examples/model_zoo/vgg.py
@@ -0,0 +1,347 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+VGG for ImageNet.
+
+Introduction
+----------------
+VGG is a convolutional neural network model proposed by K. Simonyan and A. Zisserman
+from the University of Oxford in the paper "Very Deep Convolutional Networks for
+Large-Scale Image Recognition" . The model achieves 92.7% top-5 test accuracy in ImageNet,
+which is a dataset of over 14 million images belonging to 1000 classes.
+
+Download Pre-trained Model
+----------------------------
+- Model weights in this example - vgg16_weights.npz : http://www.cs.toronto.edu/~frossard/post/vgg16/
+- Model weights in this example - vgg19.npy : https://media.githubusercontent.com/media/tensorlayer/pretrained-models/master/models/
+- Caffe VGG 16 model : https://gist.github.com/ksimonyan/211839e770f7b538e2d8#file-readme-md
+- Tool to convert the Caffe models to TensorFlow's : https://github.com/ethereon/caffe-tensorflow
+
+Note
+------
+- For simplified CNN layer see "Convolutional layer (Simplified)"
+in read the docs website.
+- When feeding other images to the model be sure to properly resize or crop them
+beforehand. Distorted images might end up being misclassified. One way of safely
+feeding images of multiple sizes is by doing center cropping.
+
+"""
+
+import os
+
+import numpy as np
+
+import tensorlayer as tl
+from tensorlayer import logging
+from tensorlayer.files import assign_weights, maybe_download_and_extract
+from tensorlayer.layers import (BatchNorm, Conv2d, Dense, Flatten, Input, SequentialLayer, MaxPool2d)
+from tensorlayer.layers import Module
+
+__all__ = [
+ 'VGG',
+ 'vgg16',
+ 'vgg19',
+ 'VGG16',
+ 'VGG19',
+ # 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn',
+ # 'vgg19_bn', 'vgg19',
+]
+
+layer_names = [
+ ['conv1_1', 'conv1_2'], 'pool1', ['conv2_1', 'conv2_2'], 'pool2',
+ ['conv3_1', 'conv3_2', 'conv3_3', 'conv3_4'], 'pool3', ['conv4_1', 'conv4_2', 'conv4_3', 'conv4_4'], 'pool4',
+ ['conv5_1', 'conv5_2', 'conv5_3', 'conv5_4'], 'pool5', 'flatten', 'fc1_relu', 'fc2_relu', 'outputs'
+]
+
+cfg = {
+ 'A': [[64], 'M', [128], 'M', [256, 256], 'M', [512, 512], 'M', [512, 512], 'M', 'F', 'fc1', 'fc2', 'O'],
+ 'B': [[64, 64], 'M', [128, 128], 'M', [256, 256], 'M', [512, 512], 'M', [512, 512], 'M', 'F', 'fc1', 'fc2', 'O'],
+ 'D':
+ [
+ [64, 64], 'M', [128, 128], 'M', [256, 256, 256], 'M', [512, 512, 512], 'M', [512, 512, 512], 'M', 'F',
+ 'fc1', 'fc2', 'O'
+ ],
+ 'E':
+ [
+ [64, 64], 'M', [128, 128], 'M', [256, 256, 256, 256], 'M', [512, 512, 512, 512], 'M', [512, 512, 512, 512],
+ 'M', 'F', 'fc1', 'fc2', 'O'
+ ],
+}
+
+mapped_cfg = {
+ 'vgg11': 'A',
+ 'vgg11_bn': 'A',
+ 'vgg13': 'B',
+ 'vgg13_bn': 'B',
+ 'vgg16': 'D',
+ 'vgg16_bn': 'D',
+ 'vgg19': 'E',
+ 'vgg19_bn': 'E'
+}
+
+model_urls = {
+ 'vgg16': 'http://www.cs.toronto.edu/~frossard/vgg16/',
+ 'vgg19': 'https://media.githubusercontent.com/media/tensorlayer/pretrained-models/master/models/'
+}
+
+model_saved_name = {'vgg16': 'vgg16_weights.npz', 'vgg19': 'vgg19.npy'}
+
+
+class VGG(Module):
+
+ def __init__(self, layer_type, batch_norm=False, end_with='outputs', name=None):
+ super(VGG, self).__init__(name=name)
+ self.end_with = end_with
+
+ config = cfg[mapped_cfg[layer_type]]
+ self.make_layer = make_layers(config, batch_norm, end_with)
+
+ def forward(self, inputs):
+ """
+ inputs : tensor
+ Shape [None, 224, 224, 3], value range [0, 1].
+ """
+
+ inputs = inputs * 255 - np.array([123.68, 116.779, 103.939], dtype=np.float32).reshape([1, 1, 1, 3])
+ out = self.make_layer(inputs)
+ return out
+
+
+def make_layers(config, batch_norm=False, end_with='outputs'):
+ layer_list = []
+ is_end = False
+ for layer_group_idx, layer_group in enumerate(config):
+ if isinstance(layer_group, list):
+ for idx, layer in enumerate(layer_group):
+ layer_name = layer_names[layer_group_idx][idx]
+ n_filter = layer
+ if idx == 0:
+ if layer_group_idx > 0:
+ in_channels = config[layer_group_idx - 2][-1]
+ else:
+ in_channels = 3
+ else:
+ in_channels = layer_group[idx - 1]
+ layer_list.append(
+ Conv2d(
+ n_filter=n_filter, filter_size=(3, 3), strides=(1, 1), act=tl.ReLU, padding='SAME',
+ in_channels=in_channels, name=layer_name
+ )
+ )
+ if batch_norm:
+ layer_list.append(BatchNorm(num_features=n_filter))
+ if layer_name == end_with:
+ is_end = True
+ break
+ else:
+ layer_name = layer_names[layer_group_idx]
+ if layer_group == 'M':
+ layer_list.append(MaxPool2d(filter_size=(2, 2), strides=(2, 2), padding='SAME', name=layer_name))
+ elif layer_group == 'O':
+ layer_list.append(Dense(n_units=1000, in_channels=4096, name=layer_name))
+ elif layer_group == 'F':
+ layer_list.append(Flatten(name='flatten'))
+ elif layer_group == 'fc1':
+ layer_list.append(Dense(n_units=4096, act=tl.ReLU, in_channels=512 * 7 * 7, name=layer_name))
+ elif layer_group == 'fc2':
+ layer_list.append(Dense(n_units=4096, act=tl.ReLU, in_channels=4096, name=layer_name))
+ if layer_name == end_with:
+ is_end = True
+ if is_end:
+ break
+ return SequentialLayer(layer_list)
+
+def restore_model(model, layer_type):
+ logging.info("Restore pre-trained weights")
+ # download weights
+ maybe_download_and_extract(model_saved_name[layer_type], 'model', model_urls[layer_type])
+ weights = []
+ if layer_type == 'vgg16':
+ npz = np.load(os.path.join('model', model_saved_name[layer_type]), allow_pickle=True)
+ # get weight list
+ for val in sorted(npz.items()):
+ logging.info(" Loading weights %s in %s" % (str(val[1].shape), val[0]))
+ weights.append(val[1])
+ if len(model.all_weights) == len(weights):
+ break
+ elif layer_type == 'vgg19':
+ npz = np.load(os.path.join('model', model_saved_name[layer_type]), allow_pickle=True, encoding='latin1').item()
+ # get weight list
+ for val in sorted(npz.items()):
+ logging.info(" Loading %s in %s" % (str(val[1][0].shape), val[0]))
+ logging.info(" Loading %s in %s" % (str(val[1][1].shape), val[0]))
+ weights.extend(val[1])
+ if len(model.all_weights) == len(weights):
+ break
+ # assign weight values
+ assign_weights(weights, model)
+ del weights
+
+def vgg16(pretrained=False, end_with='outputs', mode='dynamic', name=None):
+ """Pre-trained VGG16 model.
+
+ Parameters
+ ------------
+ pretrained : boolean
+ Whether to load pretrained weights. Default False.
+ end_with : str
+ The end point of the model. Default ``fc3_relu`` i.e. the whole model.
+ mode : str.
+ Model building mode, 'dynamic' or 'static'. Default 'dynamic'.
+ name : None or str
+ A unique layer name.
+
+ Examples
+ ---------
+ Classify ImageNet classes with VGG16, see `tutorial_models_vgg.py `__
+ With TensorLayer
+ TODO Modify the usage example according to the model storage location
+ >>> # get the whole model, without pre-trained VGG parameters
+ >>> vgg = tl.models.vgg16()
+ >>> # get the whole model, restore pre-trained VGG parameters
+ >>> vgg = tl.models.vgg16(pretrained=True)
+ >>> # use for inferencing
+ >>> output = vgg(img, is_train=False)
+ >>> probs = tf.nn.softmax(output)[0].numpy()
+
+ Extract features with VGG16 and Train a classifier with 100 classes
+
+ >>> # get VGG without the last layer
+ >>> cnn = tl.models.vgg16(end_with='fc2_relu', mode='static').as_layer()
+ >>> # add one more layer and build a new model
+ >>> ni = Input([None, 224, 224, 3], name="inputs")
+ >>> nn = cnn(ni)
+ >>> nn = tl.layers.Dense(n_units=100, name='out')(nn)
+ >>> model = tl.models.Model(inputs=ni, outputs=nn)
+ >>> # train your own classifier (only update the last layer)
+ >>> train_params = model.get_layer('out').trainable_weights
+
+ Reuse model
+
+ >>> # in dynamic model, we can directly use the same model
+ >>> # in static model
+ >>> vgg_layer = tl.models.vgg16().as_layer()
+ >>> ni_1 = tl.layers.Input([None, 224, 244, 3])
+ >>> ni_2 = tl.layers.Input([None, 224, 244, 3])
+ >>> a_1 = vgg_layer(ni_1)
+ >>> a_2 = vgg_layer(ni_2)
+ >>> M = Model(inputs=[ni_1, ni_2], outputs=[a_1, a_2])
+
+ """
+ if mode == 'dynamic':
+ model = VGG(layer_type='vgg16', batch_norm=False, end_with=end_with, name=name)
+ elif mode == 'static':
+ raise NotImplementedError
+ else:
+ raise Exception("No such mode %s" % mode)
+ if pretrained:
+ restore_model(model, layer_type='vgg16')
+ return model
+
+
+def vgg19(pretrained=False, end_with='outputs', mode='dynamic', name=None):
+ """Pre-trained VGG19 model.
+
+ Parameters
+ ------------
+ pretrained : boolean
+ Whether to load pretrained weights. Default False.
+ end_with : str
+ The end point of the model. Default ``fc3_relu`` i.e. the whole model.
+ mode : str.
+ Model building mode, 'dynamic' or 'static'. Default 'dynamic'.
+ name : None or str
+ A unique layer name.
+
+ Examples
+ ---------
+ Classify ImageNet classes with VGG19, see `tutorial_models_vgg.py `__
+ With TensorLayer
+
+ >>> # get the whole model, without pre-trained VGG parameters
+ >>> vgg = tl.models.vgg19()
+ >>> # get the whole model, restore pre-trained VGG parameters
+ >>> vgg = tl.models.vgg19(pretrained=True)
+ >>> # use for inferencing
+ >>> output = vgg(img, is_train=False)
+ >>> probs = tf.nn.softmax(output)[0].numpy()
+
+ Extract features with VGG19 and Train a classifier with 100 classes
+
+ >>> # get VGG without the last layer
+ >>> cnn = tl.models.vgg19(end_with='fc2_relu', mode='static').as_layer()
+ >>> # add one more layer and build a new model
+ >>> ni = Input([None, 224, 224, 3], name="inputs")
+ >>> nn = cnn(ni)
+ >>> nn = tl.layers.Dense(n_units=100, name='out')(nn)
+ >>> model = tl.models.Model(inputs=ni, outputs=nn)
+ >>> # train your own classifier (only update the last layer)
+ >>> train_params = model.get_layer('out').trainable_weights
+
+ Reuse model
+
+ >>> # in dynamic model, we can directly use the same model
+ >>> # in static model
+ >>> vgg_layer = tl.models.vgg19().as_layer()
+ >>> ni_1 = tl.layers.Input([None, 224, 244, 3])
+ >>> ni_2 = tl.layers.Input([None, 224, 244, 3])
+ >>> a_1 = vgg_layer(ni_1)
+ >>> a_2 = vgg_layer(ni_2)
+ >>> M = Model(inputs=[ni_1, ni_2], outputs=[a_1, a_2])
+
+ """
+ if mode == 'dynamic':
+ model = VGG(layer_type='vgg19', batch_norm=False, end_with=end_with, name=name)
+ elif mode == 'static':
+ raise NotImplementedError
+ else:
+ raise Exception("No such mode %s" % mode)
+ if pretrained:
+ restore_model(model, layer_type='vgg19')
+ return model
+
+
+VGG16 = vgg16
+VGG19 = vgg19
+
+# models without pretrained parameters
+# def vgg11(pretrained=False, end_with='outputs'):
+# model = VGG(layer_type='vgg11', batch_norm=False, end_with=end_with)
+# if pretrained:
+# model.restore_weights()
+# return model
+#
+#
+# def vgg11_bn(pretrained=False, end_with='outputs'):
+# model = VGG(layer_type='vgg11_bn', batch_norm=True, end_with=end_with)
+# if pretrained:
+# model.restore_weights()
+# return model
+#
+#
+# def vgg13(pretrained=False, end_with='outputs'):
+# model = VGG(layer_type='vgg13', batch_norm=False, end_with=end_with)
+# if pretrained:
+# model.restore_weights()
+# return model
+#
+#
+# def vgg13_bn(pretrained=False, end_with='outputs'):
+# model = VGG(layer_type='vgg13_bn', batch_norm=True, end_with=end_with)
+# if pretrained:
+# model.restore_weights()
+# return model
+#
+#
+# def vgg16_bn(pretrained=False, end_with='outputs'):
+# model = VGG(layer_type='vgg16_bn', batch_norm=True, end_with=end_with)
+# if pretrained:
+# model.restore_weights()
+# return model
+#
+#
+# def vgg19_bn(pretrained=False, end_with='outputs'):
+# model = VGG(layer_type='vgg19_bn', batch_norm=True, end_with=end_with)
+# if pretrained:
+# model.restore_weights()
+# return model
diff --git a/examples/model_zoo/yolo.py b/examples/model_zoo/yolo.py
new file mode 100644
index 0000000..d3784b2
--- /dev/null
+++ b/examples/model_zoo/yolo.py
@@ -0,0 +1,376 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""YOLOv4 for MS-COCO.
+
+# Reference:
+- [tensorflow-yolov4-tflite](
+ https://github.com/hunglc007/tensorflow-yolov4-tflite)
+
+"""
+
+import numpy as np
+import tensorlayer as tl
+from tensorlayer.layers.activation import Mish
+from tensorlayer.layers import Conv2d, MaxPool2d, BatchNorm2d, ZeroPad2d, UpSampling2d, Concat, Elementwise
+from tensorlayer.layers import Module, SequentialLayer
+from tensorlayer import logging
+
+INPUT_SIZE = 416
+weights_url = {'link': 'https://pan.baidu.com/s/1MC1dmEwpxsdgHO1MZ8fYRQ', 'password': 'idsz'}
+
+
+class Convolutional(Module):
+ """
+ Create Convolution layer
+ Because it is only a stack of reference layers, there is no build, so self._built=True
+ """
+ def __init__(self, filters_shape, downsample=False, activate=True, bn=True, activate_type='leaky',name=None):
+ super(Convolutional, self).__init__()
+ self.act = activate
+ self.act_type = activate_type
+ self.downsample = downsample
+ self.bn = bn
+ self._built = True
+ if downsample:
+ padding = 'VALID'
+ strides = 2
+ else:
+ strides = 1
+ padding = 'SAME'
+
+ if bn:
+ b_init = None
+ else:
+ b_init = tl.initializers.constant(value=0.0)
+
+ self.zeropad = ZeroPad2d(((1, 0), (1, 0)))
+ self.conv = Conv2d(n_filter=filters_shape[-1], in_channels=filters_shape[2], filter_size=(filters_shape[0], filters_shape[1]),
+ strides=(strides, strides),padding=padding, b_init=b_init, name=name)
+
+ if bn:
+ if activate == True:
+ if activate_type == 'leaky':
+ self.batchnorm2d = BatchNorm2d(act='leaky_relu0.1', num_features=filters_shape[-1])
+ elif activate_type == 'mish':
+ self.batchnorm2d = BatchNorm2d(act=Mish, num_features=filters_shape[-1])
+ else:
+ self.batchnorm2d = BatchNorm2d(act=None, num_features=filters_shape[-1])
+
+ def forward(self, input):
+ if self.downsample:
+ input = self.zeropad(input)
+
+ output = self.conv(input)
+
+ if self.bn:
+ output = self.batchnorm2d(output)
+ return output
+
+class residual_block(Module):
+ def __init__(self, input_channel, filter_num1, filter_num2, activate_type='leaky'):
+ super(residual_block, self).__init__()
+ self.conv1 = Convolutional(filters_shape=(1, 1, input_channel, filter_num1), activate_type=activate_type)
+ self.conv2 = Convolutional(filters_shape=(3, 3, filter_num1, filter_num2), activate_type=activate_type)
+ self.add = Elementwise(tl.add)
+
+ def forward(self, inputs):
+ output = self.conv1(inputs)
+ output = self.conv2(output)
+ output = self.add([inputs, output])
+ return output
+
+def residual_block_num(num, input_channel, filter_num1, filter_num2, activate_type='leaky'):
+ residual_list = []
+ for i in range(num):
+ residual_list.append(residual_block(input_channel, filter_num1, filter_num2, activate_type=activate_type))
+ return SequentialLayer(residual_list)
+
+class cspdarknet53(Module):
+ def __init__(self):
+ super(cspdarknet53, self).__init__()
+ self._built = True
+ self.conv1_1 = Convolutional((3, 3, 3, 32), activate_type='mish')
+ self.conv1_2 = Convolutional((3, 3, 32, 64), downsample=True, activate_type='mish')
+ self.conv1_3 = Convolutional((1, 1, 64, 64), activate_type='mish', name='conv_rote_block_1')
+ self.conv1_4 = Convolutional((1, 1, 64, 64), activate_type='mish')
+ self.residual_1 = residual_block_num(1, 64, 32, 64, activate_type="mish")
+
+ self.conv2_1 = Convolutional((1, 1, 64, 64), activate_type='mish')
+ self.concat = Concat()
+ self.conv2_2 = Convolutional((1, 1, 128, 64), activate_type='mish')
+ self.conv2_3 = Convolutional((3, 3, 64, 128), downsample=True, activate_type='mish')
+ self.conv2_4 = Convolutional((1, 1, 128, 64), activate_type='mish', name='conv_rote_block_2')
+ self.conv2_5 = Convolutional((1, 1, 128, 64), activate_type='mish')
+ self.residual_2 = residual_block_num(2, 64, 64, 64, activate_type='mish')
+
+ self.conv3_1 = Convolutional((1, 1, 64, 64), activate_type='mish')
+ self.conv3_2 = Convolutional((1, 1, 128, 128), activate_type='mish')
+ self.conv3_3 = Convolutional((3, 3, 128, 256), downsample=True, activate_type='mish')
+ self.conv3_4 = Convolutional((1, 1, 256, 128), activate_type='mish', name='conv_rote_block_3')
+ self.conv3_5 = Convolutional((1, 1, 256, 128), activate_type='mish')
+ self.residual_3 = residual_block_num(8, 128, 128, 128, activate_type="mish")
+
+ self.conv4_1 = Convolutional((1, 1, 128, 128), activate_type='mish')
+ self.conv4_2 = Convolutional((1, 1, 256, 256), activate_type='mish')
+ self.conv4_3 = Convolutional((3, 3, 256, 512), downsample=True, activate_type='mish')
+ self.conv4_4 = Convolutional((1, 1, 512, 256), activate_type='mish', name='conv_rote_block_4')
+ self.conv4_5 = Convolutional((1, 1, 512, 256), activate_type='mish')
+ self.residual_4 = residual_block_num(8, 256, 256, 256, activate_type="mish")
+
+ self.conv5_1 = Convolutional((1, 1, 256, 256), activate_type='mish')
+ self.conv5_2 = Convolutional((1, 1, 512, 512), activate_type='mish')
+ self.conv5_3 = Convolutional((3, 3, 512, 1024), downsample=True, activate_type='mish')
+ self.conv5_4 = Convolutional((1, 1, 1024, 512), activate_type='mish', name='conv_rote_block_5')
+ self.conv5_5 = Convolutional((1, 1, 1024, 512), activate_type='mish')
+ self.residual_5 = residual_block_num(4, 512, 512, 512, activate_type="mish")
+
+
+ self.conv6_1 = Convolutional((1, 1, 512, 512), activate_type='mish')
+ self.conv6_2 = Convolutional((1, 1, 1024, 1024), activate_type='mish')
+ self.conv6_3 = Convolutional((1, 1, 1024, 512))
+ self.conv6_4 = Convolutional((3, 3, 512, 1024))
+ self.conv6_5 = Convolutional((1, 1, 1024, 512))
+
+ self.maxpool1 = MaxPool2d(filter_size=(13, 13), strides=(1, 1))
+ self.maxpool2 = MaxPool2d(filter_size=(9, 9), strides=(1, 1))
+ self.maxpool3 = MaxPool2d(filter_size=(5, 5), strides=(1, 1))
+
+ self.conv7_1 = Convolutional((1, 1, 2048, 512))
+ self.conv7_2 = Convolutional((3, 3, 512, 1024))
+ self.conv7_3 = Convolutional((1, 1, 1024, 512))
+
+ def forward(self, input_data):
+ input_data = self.conv1_1(input_data)
+ input_data = self.conv1_2(input_data)
+ route = input_data
+ route = self.conv1_3(route)
+ input_data = self.conv1_4(input_data)
+ input_data = self.residual_1(input_data)
+
+ input_data = self.conv2_1(input_data)
+ input_data = self.concat([input_data, route])
+ input_data = self.conv2_2(input_data)
+ input_data = self.conv2_3(input_data)
+ route = input_data
+ route = self.conv2_4(route)
+ input_data = self.conv2_5(input_data)
+ input_data = self.residual_2(input_data)
+
+ input_data = self.conv3_1(input_data)
+ input_data = self.concat([input_data, route])
+ input_data = self.conv3_2(input_data)
+ input_data = self.conv3_3(input_data)
+ route = input_data
+ route = self.conv3_4(route)
+ input_data = self.conv3_5(input_data)
+ input_data = self.residual_3(input_data)
+
+ input_data = self.conv4_1(input_data)
+ input_data = self.concat([input_data, route])
+ input_data = self.conv4_2(input_data)
+ route_1 = input_data
+ input_data = self.conv4_3(input_data)
+ route = input_data
+ route = self.conv4_4(route)
+ input_data = self.conv4_5(input_data)
+ input_data = self.residual_4(input_data)
+
+ input_data = self.conv5_1(input_data)
+ input_data = self.concat([input_data, route])
+ input_data = self.conv5_2(input_data)
+ route_2 = input_data
+ input_data = self.conv5_3(input_data)
+ route = input_data
+ route = self.conv5_4(route)
+ input_data = self.conv5_5(input_data)
+ input_data = self.residual_5(input_data)
+
+ input_data = self.conv6_1(input_data)
+ input_data = self.concat([input_data, route])
+
+ input_data = self.conv6_2(input_data)
+ input_data = self.conv6_3(input_data)
+ input_data = self.conv6_4(input_data)
+ input_data = self.conv6_5(input_data)
+
+ maxpool1 = self.maxpool1(input_data)
+ maxpool2 = self.maxpool2(input_data)
+ maxpool3 = self.maxpool3(input_data)
+ input_data = self.concat([maxpool1, maxpool2, maxpool3, input_data])
+
+ input_data = self.conv7_1(input_data)
+ input_data = self.conv7_2(input_data)
+ input_data = self.conv7_3(input_data)
+
+ return route_1, route_2, input_data
+
+
+class YOLOv4_model(Module):
+ def __init__(self, NUM_CLASS):
+ super(YOLOv4_model, self).__init__()
+ self.cspdarnnet = cspdarknet53()
+
+ self.conv1_1 = Convolutional((1, 1, 512, 256))
+ self.upsamle = UpSampling2d(scale=2)
+ self.conv1_2 = Convolutional((1, 1, 512, 256), name='conv_yolo_1')
+ self.concat = Concat()
+
+ self.conv2_1 = Convolutional((1, 1, 512, 256))
+ self.conv2_2 = Convolutional((3, 3, 256, 512))
+ self.conv2_3 = Convolutional((1, 1, 512, 256))
+ self.conv2_4 = Convolutional((3, 3, 256, 512))
+ self.conv2_5 = Convolutional((1, 1, 512, 256))
+
+ self.conv3_1 = Convolutional((1, 1, 256, 128))
+ self.conv3_2 = Convolutional((1, 1, 256, 128), name='conv_yolo_2')
+
+ self.conv4_1 = Convolutional((1, 1, 256, 128))
+ self.conv4_2 = Convolutional((3, 3, 128, 256))
+ self.conv4_3 = Convolutional((1, 1, 256, 128))
+ self.conv4_4 = Convolutional((3, 3, 128, 256))
+ self.conv4_5 = Convolutional((1, 1, 256, 128))
+
+ self.conv5_1 = Convolutional((3, 3, 128, 256), name='conv_route_1')
+ self.conv5_2 = Convolutional((1, 1, 256, 3 * (NUM_CLASS + 5)), activate=False, bn=False)
+
+ self.conv6_1 = Convolutional((3, 3, 128, 256), downsample=True, name='conv_route_2')
+ self.conv6_2 = Convolutional((1, 1, 512, 256))
+ self.conv6_3 = Convolutional((3, 3, 256, 512))
+ self.conv6_4 = Convolutional((1, 1, 512, 256))
+ self.conv6_5 = Convolutional((3, 3, 256, 512))
+ self.conv6_6 = Convolutional((1, 1, 512, 256))
+
+ self.conv7_1 = Convolutional((3, 3, 256, 512), name='conv_route_3')
+ self.conv7_2 = Convolutional((1, 1, 512, 3 * (NUM_CLASS + 5)), activate=False, bn=False)
+ self.conv7_3 = Convolutional((3, 3, 256, 512), downsample=True, name='conv_route_4')
+
+ self.conv8_1 = Convolutional((1, 1, 1024, 512))
+ self.conv8_2 = Convolutional((3, 3, 512, 1024))
+ self.conv8_3 = Convolutional((1, 1, 1024, 512))
+ self.conv8_4 = Convolutional((3, 3, 512, 1024))
+ self.conv8_5 = Convolutional((1, 1, 1024, 512))
+
+ self.conv9_1 = Convolutional((3, 3, 512, 1024))
+ self.conv9_2 = Convolutional((1, 1, 1024, 3 * (NUM_CLASS + 5)), activate=False, bn=False)
+
+ def forward(self, inputs):
+ route_1, route_2, conv = self.cspdarnnet(inputs)
+
+ route = conv
+ conv = self.conv1_1(conv)
+ conv = self.upsamle(conv)
+ route_2 = self.conv1_2(route_2)
+ conv = self.concat([route_2, conv])
+
+ conv = self.conv2_1(conv)
+ conv = self.conv2_2(conv)
+ conv = self.conv2_3(conv)
+ conv = self.conv2_4(conv)
+ conv = self.conv2_5(conv)
+
+ route_2 = conv
+ conv = self.conv3_1(conv)
+ conv = self.upsamle(conv)
+ route_1 = self.conv3_2(route_1)
+ conv = self.concat([route_1, conv])
+
+ conv = self.conv4_1(conv)
+ conv = self.conv4_2(conv)
+ conv = self.conv4_3(conv)
+ conv = self.conv4_4(conv)
+ conv = self.conv4_5(conv)
+
+ route_1 = conv
+ conv = self.conv5_1(conv)
+ conv_sbbox = self.conv5_2(conv)
+
+ conv = self.conv6_1(route_1)
+ conv = self.concat([conv, route_2])
+
+ conv = self.conv6_2(conv)
+ conv = self.conv6_3(conv)
+ conv = self.conv6_4(conv)
+ conv = self.conv6_5(conv)
+ conv = self.conv6_6(conv)
+
+ route_2 = conv
+ conv = self.conv7_1(conv)
+ conv_mbbox = self.conv7_2(conv)
+ conv = self.conv7_3(route_2)
+ conv = self.concat([conv, route])
+
+ conv = self.conv8_1(conv)
+ conv = self.conv8_2(conv)
+ conv = self.conv8_3(conv)
+ conv = self.conv8_4(conv)
+ conv = self.conv8_5(conv)
+
+ conv = self.conv9_1(conv)
+ conv_lbbox = self.conv9_2(conv)
+
+ return conv_sbbox, conv_mbbox, conv_lbbox
+
+def YOLOv4(NUM_CLASS, pretrained=False):
+ """Pre-trained YOLOv4 model.
+
+ Parameters
+ ------------
+ NUM_CLASS : int
+ Number of classes in final prediction.
+ pretrained : boolean
+ Whether to load pretrained weights. Default False.
+
+ Examples
+ ---------
+ Object Detection with YOLOv4, see `computer_vision.py
+ `__
+ With TensorLayer
+
+ >>> # get the whole model, without pre-trained YOLOv4 parameters
+ >>> yolov4 = tl.app.YOLOv4(NUM_CLASS=80, pretrained=False)
+ >>> # get the whole model, restore pre-trained YOLOv4 parameters
+ >>> yolov4 = tl.app.YOLOv4(NUM_CLASS=80, pretrained=True)
+ >>> # use for inferencing
+ >>> output = yolov4(img, is_train=False)
+
+ """
+
+ network = YOLOv4_model(NUM_CLASS=NUM_CLASS)
+
+ if pretrained:
+ restore_params(network, model_path='model/yolov4_model.npz')
+
+ return network
+
+
+def restore_params(network, model_path='models.npz'):
+ logging.info("Restore pre-trained weights")
+
+ try:
+ npz = np.load(model_path, allow_pickle=True)
+ except:
+ print("Download the model file, placed in the /model ")
+ print("Weights download: ", weights_url['link'], "password:", weights_url['password'])
+
+ txt_path = 'model/yolov4_weights3_config.txt'
+ f = open(txt_path, "r")
+ line = f.readlines()
+ for i in range(len(line)):
+ network.all_weights[i].assign(npz[line[i].strip()])
+ logging.info(" Loading weights %s in %s" % (network.all_weights[i].shape, network.all_weights[i].name))
+
+def tl2_weights_to_tl3_weights(weights_2_path='model/weights_2.txt', weights_3_path='model/weights_3.txt', txt_path='model/yolov4_weights_config.txt'):
+ weights_2_path = weights_2_path
+ weights_3_path = weights_3_path
+ txt_path = txt_path
+ f1 = open(weights_2_path, "r")
+ f2 = open(weights_3_path, "r")
+ f3 = open(txt_path, "r")
+ line1 = f1.readlines()
+ line2 = f2.readlines()
+ line3 = f3.readlines()
+ _dicts = {}
+ for i in range(len(line1)):
+ _dicts[line1[i].strip()] = line3[i].strip()
+ for j in range(len(line2)):
+ print(_dicts[line2[j].strip()])
diff --git a/img/TL_gardener.png b/img/TL_gardener.png
new file mode 100644
index 0000000..ca60068
Binary files /dev/null and b/img/TL_gardener.png differ
diff --git a/img/TL_gardener.psd b/img/TL_gardener.psd
new file mode 100644
index 0000000..e45b815
Binary files /dev/null and b/img/TL_gardener.psd differ
diff --git a/img/awesome-mentioned.png b/img/awesome-mentioned.png
new file mode 100644
index 0000000..1bbc400
Binary files /dev/null and b/img/awesome-mentioned.png differ
diff --git a/img/github_mascot.png b/img/github_mascot.png
new file mode 100644
index 0000000..a93ba99
Binary files /dev/null and b/img/github_mascot.png differ
diff --git a/img/img_tensorflow.png b/img/img_tensorflow.png
new file mode 100644
index 0000000..a8190c9
Binary files /dev/null and b/img/img_tensorflow.png differ
diff --git a/img/img_tensorlayer.png b/img/img_tensorlayer.png
new file mode 100644
index 0000000..c6b1f7c
Binary files /dev/null and b/img/img_tensorlayer.png differ
diff --git a/img/img_tlayer1.png b/img/img_tlayer1.png
new file mode 100644
index 0000000..7b998ea
Binary files /dev/null and b/img/img_tlayer1.png differ
diff --git a/img/join_slack.png b/img/join_slack.png
new file mode 100644
index 0000000..96e5902
Binary files /dev/null and b/img/join_slack.png differ
diff --git a/img/join_slack.psd b/img/join_slack.psd
new file mode 100644
index 0000000..0716f60
Binary files /dev/null and b/img/join_slack.psd differ
diff --git a/img/medium/Readme.md b/img/medium/Readme.md
new file mode 100644
index 0000000..35ea25a
--- /dev/null
+++ b/img/medium/Readme.md
@@ -0,0 +1,19 @@
+# Medium GuideLines
+
+## Publication Avatar
+
+
+
+## Publication Logo (appears on posts)
+
+
+
+
+Add a publication logo, which appears at the top of all your publication's stories.
+
+It is 72px tall and can have a maximum width of 600px.
+
+## Publication homepage images
+
+Under Homepage and settings > Layout, you can select a header size, upload a logo and add a background image (large header size only).
+
diff --git a/img/medium/medium_header.png b/img/medium/medium_header.png
new file mode 100644
index 0000000..87bebf3
Binary files /dev/null and b/img/medium/medium_header.png differ
diff --git a/img/medium/medium_header.psd b/img/medium/medium_header.psd
new file mode 100644
index 0000000..9212a0a
Binary files /dev/null and b/img/medium/medium_header.psd differ
diff --git a/img/tl_black_logo.png b/img/tl_black_logo.png
new file mode 100644
index 0000000..8bf892d
Binary files /dev/null and b/img/tl_black_logo.png differ
diff --git a/img/tl_transparent_logo.png b/img/tl_transparent_logo.png
new file mode 100644
index 0000000..359a232
Binary files /dev/null and b/img/tl_transparent_logo.png differ
diff --git a/img/tl_white_logo.png b/img/tl_white_logo.png
new file mode 100644
index 0000000..a42806d
Binary files /dev/null and b/img/tl_white_logo.png differ
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
new file mode 100644
index 0000000..f251c65
--- /dev/null
+++ b/requirements/requirements.txt
@@ -0,0 +1,10 @@
+imageio>=2.5.0
+numpy>=1.16
+progressbar2>=3.39.3
+requests>=2.21.0
+scikit-learn>=0.21.0
+scikit-image>=0.15.0
+scipy>=1.2.1
+wrapt>=1.11.1
+h5py>=2.9
+cloudpickle>=0.8.1
diff --git a/requirements/requirements_contrib_loggers.txt b/requirements/requirements_contrib_loggers.txt
new file mode 100644
index 0000000..245d842
--- /dev/null
+++ b/requirements/requirements_contrib_loggers.txt
@@ -0,0 +1 @@
+hyperdash>=0.15,<0.16
\ No newline at end of file
diff --git a/requirements/requirements_db.txt b/requirements/requirements_db.txt
new file mode 100644
index 0000000..f9cb647
--- /dev/null
+++ b/requirements/requirements_db.txt
@@ -0,0 +1 @@
+pymongo>=3.8.0
diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt
new file mode 100644
index 0000000..ef32fc1
--- /dev/null
+++ b/requirements/requirements_dev.txt
@@ -0,0 +1 @@
+autopep8>=1.3,<1.5
diff --git a/requirements/requirements_doc.txt b/requirements/requirements_doc.txt
new file mode 100644
index 0000000..02d813a
--- /dev/null
+++ b/requirements/requirements_doc.txt
@@ -0,0 +1,8 @@
+flake8-docstrings>=1.3,<1.4
+pycodestyle>=2.5.0
+pydocstyle>=2.1,<3.1
+sphinx==2.0.1
+sphinx_rtd_theme>=0.4,<0.5
+wrapt>=1.11.1
+h5py>=2.9
+cloudpickle>=0.8.1
diff --git a/requirements/requirements_extra.txt b/requirements/requirements_extra.txt
new file mode 100644
index 0000000..3a818c6
--- /dev/null
+++ b/requirements/requirements_extra.txt
@@ -0,0 +1,6 @@
+opencv-python>=4.1.0.25
+nltk>=3.3,<3.5
+matplotlib>=2.2,<3.1
+requests>=2.21.0
+tqdm>=4.31.1
+lxml>=4.3.3
diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt
new file mode 100644
index 0000000..e47c0ed
--- /dev/null
+++ b/requirements/requirements_test.txt
@@ -0,0 +1,9 @@
+keras>=2.2,<2.3
+pycodestyle>=2.5.0
+pydocstyle>=2.1,<3.1
+pytest>=4.5.0
+pytest-cache>=1.0,<1.1
+pytest-cov>=2.7.1
+pytest-xdist>=1.28.0
+sphinx==2.0.1
+yapf>=0.27.0
diff --git a/requirements/requirements_tf_cpu.txt b/requirements/requirements_tf_cpu.txt
new file mode 100644
index 0000000..72e758e
--- /dev/null
+++ b/requirements/requirements_tf_cpu.txt
@@ -0,0 +1 @@
+tensorflow>=2.0.0-alpha0
diff --git a/requirements/requirements_tf_gpu.txt b/requirements/requirements_tf_gpu.txt
new file mode 100644
index 0000000..25247e0
--- /dev/null
+++ b/requirements/requirements_tf_gpu.txt
@@ -0,0 +1 @@
+tensorflow-gpu>=2.0.0-alpha0
diff --git a/run_compile.py b/run_compile.py
new file mode 100644
index 0000000..495272d
--- /dev/null
+++ b/run_compile.py
@@ -0,0 +1,74 @@
+import tensorlayer as T
+from dragon.vm.tensorlayer.layers import Dense
+from dragon.vm.tensorlayer.models import Model
+import dragon.vm.tensorlayer as tl
+
+import dragon as dg
+import argparse
+import numpy as np
+
+X_train, y_train, X_val, y_val, X_test, y_test = T.files.load_mnist_dataset(shape=(-1, 784))
+
+
+class MLP(Model):
+
+ def __init__(self):
+ super(MLP, self).__init__()
+ self.dense1 = Dense(n_units=800, act=tl.act.relu, in_channels=784)
+ self.dense2 = Dense(n_units=800, act=tl.act.relu, in_channels=800)
+ self.dense3 = Dense(n_units=10, act=tl.act.relu, in_channels=800)
+
+ def forward(self, x):
+ z = self.dense1(x)
+ z = self.dense2(z)
+ out = self.dense3(z)
+ return out
+
+
+class Classifier(object):
+ """The base classifier class."""
+
+ # TensorSpec for graph execution
+ image_spec = dg.Tensor([None, 3, 32, 32], 'float32')
+ label_spec = dg.Tensor([None], 'int64')
+
+ def __init__(self, optimizer):
+ super(Classifier, self).__init__()
+ self.net = MLP()
+ self.optimizer = optimizer
+ self.params = self.net.trainable_weights
+
+ def step(self, image, label):
+ with dg.GradientTape() as tape:
+ logit = self.net(image)
+ # logit = dg.cast(logit, 'float64')
+ logit = dg.cast(dg.math.argmax(logit, -1), 'int32')
+ # label = dg.cast(label, 'float32')
+ # print("logit :\n", logit, label)
+ # loss = dg.losses.smooth_l1_loss([logit, label])
+ # loss = tl.losses.sparse_softmax_crossentropy(logit, label)
+ loss = dg.math.sum(
+ (logit - label) * (logit - label)
+ ) # dg.losses.sparse_softmax_cross_entropy([logit, label])
+ accuracy = dg.math.mean(dg.math.equal([logit, label]).astype('float32'))
+ grads = tape.gradient(loss, self.params)
+ self.optimizer.apply_gradients(zip(self.params, grads))
+ return loss, accuracy, self.optimizer
+
+
+if __name__ == '__main__':
+ dg.autograph.set_execution('EAGER_MODE')
+ # Define the model
+ model = Classifier(dg.optimizers.SGD(base_lr=0.001, momentum=0.9, weight_decay=1e-4))
+ # Main loop
+ batch_size = 200
+ for i in range(50):
+ for X_batch, y_batch in T.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
+ image = dg.EagerTensor(X_batch, copy=False)
+ label = dg.EagerTensor(y_batch, copy=False, dtype='float32')
+ loss, accuracy, _ = model.step(image, label)
+ if i % 20 == 0:
+ dg.logging.info(
+ 'Iteration %d, lr = %s, loss = %.5f, accuracy = %.3f' %
+ (i, str(model.optimizer.base_lr), loss, accuracy)
+ )
diff --git a/scripts/download_and_install_openmpi3_ubuntu.sh b/scripts/download_and_install_openmpi3_ubuntu.sh
new file mode 100644
index 0000000..8aa233a
--- /dev/null
+++ b/scripts/download_and_install_openmpi3_ubuntu.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -e
+
+if [ $(uname) == "Darwin" ]; then
+ NPROC=$(sysctl -n hw.ncpu)
+else
+ NPROC=$(nproc)
+fi
+
+mkdir -p $HOME/openmpi_tmp && cd $HOME/openmpi_tmp
+
+# TODO: upgrade to latest version once https://github.com/open-mpi/ompi/pull/5296 is in the release
+MPI_MAJOR=3
+MPI_MINOR=1
+
+VERSION=${MPI_MAJOR}.${MPI_MINOR}.0
+FILENAME=openmpi-${VERSION}.tar.bz2
+FOLDER=openmpi-${VERSION}
+URL=https://download.open-mpi.org/release/open-mpi/v${MPI_MAJOR}.${MPI_MINOR}/${FILENAME}
+
+[ ! -f ${FILENAME} ] && curl -vLOJ $URL
+tar -xf ${FILENAME}
+cd ${FOLDER}
+
+# will take about 8 min or longer depends on your machine
+./configure --prefix=$HOME/local/openmpi
+make -j ${NPROC} all
+make install
+
+rm -rf $HOME/openmpi_tmp
+
+echo 'Update the PATH with OpenMPI bin by running: PATH=$PATH:$HOME/local/openmpi/bin'
+echo 'Update the PATH in ~/.bashrc if you want OpenMPI to be ready once the machine start'
diff --git a/scripts/install-horovod-for-doc-test.sh b/scripts/install-horovod-for-doc-test.sh
new file mode 100644
index 0000000..22c0558
--- /dev/null
+++ b/scripts/install-horovod-for-doc-test.sh
@@ -0,0 +1,15 @@
+# This script is for installing horovod on travis-CI only!
+
+set -e
+
+[ ! -z "$1" ] && export PATH=$1:$PATH
+
+mkdir -p /opt
+chmod a+rx /opt
+cd /opt
+wget https://github.com/lgarithm/openmpi-release/raw/master/releases/openmpi-bin-3.1.0.tar.bz2
+tar -xf openmpi-bin-3.1.0.tar.bz2
+
+PATH=/opt/openmpi/bin:$PATH pip install horovod
+
+echo "done $0"
diff --git a/scripts/install-requirements-for-rtd.sh b/scripts/install-requirements-for-rtd.sh
new file mode 100644
index 0000000..fd0693c
--- /dev/null
+++ b/scripts/install-requirements-for-rtd.sh
@@ -0,0 +1,28 @@
+# This script is for installing horovod on readthedocs only!
+set -e
+
+pwd
+SCRIPT_DIR=$(cd $(dirname $0) && pwd)
+
+[ ! -z "$1" ] && export PATH=$1:$PATH
+
+LOCATION=/home/docs
+URL=https://github.com/lgarithm/openmpi-release/raw/master/releases/openmpi-bin-3.1.0-rtd.tar.bz2
+
+mkdir -p ${LOCATION}
+chmod a+rx ${LOCATION}
+cd ${LOCATION}
+curl -vLOJ ${URL}
+tar -xf *.tar.bz2
+
+pip install tensorflow==1.5.0 # must install tensorflow before horovod
+PATH=${LOCATION}/openmpi/bin:$PATH pip install horovod
+
+# install all requirements except tensorflow
+for req in $(find $SCRIPT_DIR/../requirements -type f); do
+ if [ ! $(grep tensorflow $req) ]; then
+ pip install -r $req
+ fi
+done
+
+echo "done $0"
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..67f86fe
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,79 @@
+[tool:pytest]
+testpaths = tests/
+
+[flake8]
+max-line-length = 120
+ignore =
+ D301
+ E221 # Space before equal sign
+ E251 # Space after equal sign
+exclude =
+ .git,
+ venv,
+ __pycache__,
+ .pytest_cache,
+ tensorlayer.egg-info,
+ build,
+ dist,
+ img
+
+[yapf]
+based_on_style=google
+
+# The number of columns to use for indentation.
+indent_width = 4
+
+# The column limit.
+column_limit=120
+
+# Place each dictionary entry onto its own line.
+each_dict_entry_on_separate_line = True
+
+# Put closing brackets on a separate line, dedented, if the bracketed
+# expression can't fit in a single line. Applies to all kinds of brackets,
+# including function definitions and calls. For example:
+#
+# config = {
+# 'key1': 'value1',
+# 'key2': 'value2',
+# } # <--- this bracket is dedented and on a separate line
+#
+# time_series = self.remote_client.query_entity_counters(
+# entity='dev3246.region1',
+# key='dns.query_latency_tcp',
+# transform=Transformation.AVERAGE(window=timedelta(seconds=60)),
+# start_ts=now()-timedelta(days=3),
+# end_ts=now(),
+# ) # <--- this bracket is dedented and on a separate line
+dedent_closing_brackets=True
+
+# Do not split consecutive brackets. Only relevant when DEDENT_CLOSING_BRACKETS is set
+coalesce_brackets = False
+
+# Align closing bracket with visual indentation.
+align_closing_bracket_with_visual_indent = False
+
+# Split named assignments onto individual lines.
+split_before_named_assigns = False
+
+# If an argument / parameter list is going to be split, then split before the first argument.
+split_before_first_argument = True
+
+# Split before arguments if the argument list is terminated by a comma.
+split_arguments_when_comma_terminated = False
+
+# Insert a space between the ending comma and closing bracket of a list, etc.
+space_between_ending_comma_and_closing_bracket = True
+
+# Join short lines into one line. E.g., single line if statements.
+join_multiple_lines = True
+
+# Do not include spaces around selected binary operators.
+# Example: 1 + 2 * 3 - 4 / 5 => 1 + 2*3 - 4/5
+no_spaces_around_selected_binary_operators = True
+
+# Allow lambdas to be formatted on more than one line.
+allow_multiline_lambdas = True
+
+SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT = 10
+SPLIT_PENALTY_AFTER_OPENING_BRACKET = 500
\ No newline at end of file
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..76ac682
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+import codecs
+import os
+import sys
+
+os.environ['TENSORLAYER_PACKAGE_BUILDING'] = 'True'
+
+
+try:
+ from setuptools import find_packages, setup, Extension
+ from setuptools.command.build_ext import build_ext
+
+except ImportError:
+ from distutils.core import (
+ setup,
+ find_packages
+ )
+
+
+from tensorlayer import (
+ __contact_emails__,
+ __contact_names__,
+ __description__,
+ __download_url__,
+ __homepage__,
+ __keywords__,
+ __license__,
+ __package_name__,
+ __repository_url__,
+ __version__
+)
+
+
+# =================== Reading Readme file as TXT files ===================
+
+if os.path.exists('README.rst'):
+ # codec is used for consistent encoding
+ long_description = codecs.open(
+ os.path.join(os.path.abspath(os.path.dirname(__file__)), 'README.rst'),
+ 'r', 'utf-8'
+ ).read()
+
+else:
+ long_description = 'See ' + __homepage__
+
+# ======================= Reading Requirements files as TXT files =======================
+
+
+def req_file(filename, folder="requirements"):
+ with open(os.path.join(folder, filename)) as f:
+ content = f.readlines()
+ # you may also want to remove whitespace characters
+ # Example: `\n` at the end of each line
+ return [x.strip() for x in content]
+
+# ======================= Defining the requirements var =======================
+
+
+install_requires = req_file("requirements.txt")
+
+extras_require = {
+ # User packages
+ 'tf_cpu': req_file("requirements_tf_cpu.txt"),
+ 'tf_gpu': req_file("requirements_tf_gpu.txt"),
+ 'extra': req_file("requirements_extra.txt"),
+
+ # Contrib Packages
+ 'contrib_loggers': req_file("requirements_contrib_loggers.txt"),
+
+ # Dev Packages
+ 'test': req_file("requirements_test.txt"),
+ 'dev': req_file("requirements_dev.txt"),
+ 'doc': req_file("requirements_doc.txt"),
+ 'db': req_file("requirements_db.txt"),
+}
+
+extras_require['all'] = sum([extras_require.get(key) for key in ['extra', 'contrib_loggers']], list())
+
+extras_require['all_cpu'] = sum([extras_require.get(key) for key in ['all', 'tf_cpu']], list())
+extras_require['all_gpu'] = sum([extras_require.get(key) for key in ['all', 'tf_gpu']], list())
+
+extras_require['all_dev'] = sum([extras_require.get(key) for key in ['all', 'db', 'dev', 'doc', 'test']], list())
+extras_require['all_cpu_dev'] = sum([extras_require.get(key) for key in ['all_dev', 'tf_cpu']], list())
+extras_require['all_gpu_dev'] = sum([extras_require.get(key) for key in ['all_dev', 'tf_gpu']], list())
+
+
+cmdclass = dict()
+ext_modules = []
+
+
+# Readthedocs requires TF 1.5.0 to build properly
+if 'READTHEDOCS' in os.environ:
+ ext_modules = [
+ Extension('install_requirements_for_rtd', []),
+ ]
+
+ class custom_build_ext(build_ext):
+ def build_extensions(self):
+ os.system('./scripts/install-requirements-for-rtd.sh %s' %
+ os.path.dirname(sys.executable))
+
+ cmdclass = {'build_ext': custom_build_ext}
+
+
+# ======================= Define the package setup =======================
+
+
+setup(
+ name=__package_name__,
+
+ # Versions should comply with PEP440. For a discussion on single-sourcing
+ # the version across setup.py and the project code, see
+ # https://packaging.python.org/en/latest/single_source_version.html
+ version=__version__,
+
+ description=__description__,
+ long_description=long_description,
+
+ # The project's main homepage.
+ url=__repository_url__,
+ download_url=__download_url__,
+
+ # Author details
+ author=__contact_names__,
+ author_email=__contact_emails__,
+
+ # maintainer Details
+ maintainer=__contact_names__,
+ maintainer_email=__contact_emails__,
+
+ # The licence under which the project is released
+ license=__license__,
+
+ classifiers=[
+ # How mature is this project? Common values are
+ # 1 - Planning
+ # 2 - Pre-Alpha
+ # 3 - Alpha
+ # 4 - Beta
+ # 5 - Production/Stable
+ # 6 - Mature
+ # 7 - Inactive
+ 'Development Status :: 5 - Production/Stable',
+
+ # Indicate who your project is intended for
+ 'Intended Audience :: Developers',
+ 'Intended Audience :: Science/Research',
+ 'Intended Audience :: Information Technology',
+
+ # Indicate what your project relates to
+ 'Topic :: Scientific/Engineering',
+ 'Topic :: Scientific/Engineering :: Image Recognition',
+ 'Topic :: Scientific/Engineering :: Artificial Intelligence',
+ 'Topic :: Software Development :: Libraries',
+ 'Topic :: Utilities',
+
+ # Pick your license as you wish (should match "license" above)
+ 'License :: OSI Approved :: Apache Software License',
+
+ # Specify the Python versions you support here. In particular, ensure
+ # that you indicate whether you support Python 2, Python 3 or both.
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
+
+ # Additionnal Settings
+ 'Environment :: Console',
+ 'Natural Language :: English',
+ 'Operating System :: OS Independent',
+ ],
+
+ keywords=__keywords__,
+ packages=find_packages(),
+
+ # List run-time dependencies here. These will be installed by pip when
+ # your project is installed. For an analysis of "install_requires" vs pip's
+ # requirements files see:
+ # https://packaging.python.org/en/latest/requirements.html
+ install_requires=install_requires,
+
+ cmdclass=cmdclass,
+
+ # List additional groups of dependencies here (e.g. development
+ # dependencies). You can install these using the following syntax,
+ # $ pip install -e .[test]
+ extras_require=extras_require,
+ ext_modules=ext_modules,
+
+ scripts=[
+ 'tl',
+ ],
+)
diff --git a/setup.travis.cfg b/setup.travis.cfg
new file mode 100644
index 0000000..b786d1b
--- /dev/null
+++ b/setup.travis.cfg
@@ -0,0 +1,111 @@
+[tool:pytest]
+testpaths = tests/
+addopts = --ignore=tests/test_documentation.py
+ --ignore=tests/test_yapf_format.py
+ --ignore=tests/pending/test_decorators.py
+ --ignore=tests/pending/test_documentation.py
+ --ignore=tests/pending/test_logging.py
+ --ignore=tests/pending/test_pydocstyle.py
+ --ignore=tests/pending/test_layers_padding.py
+ --ignore=tests/pending/test_timeout.py
+ --ignore=tests/pending/test_layers_super_resolution.py
+ --ignore=tests/pending/test_reuse_mlp.py
+ --ignore=tests/pending/test_layers_importer.py
+ --ignore=tests/pending/test_layers_time_distributed.py
+ --ignore=tests/pending/test_layers_spatial_transformer.py
+ --ignore=tests/pending/test_layers_stack.py
+ --ignore=tests/pending/test_mnist_simple.py
+ --ignore=tests/pending/test_tf_layers.py
+ --ignore=tests/pending/test_array_ops.py
+ --ignore=tests/pending/test_layers_basic.py
+ --ignore=tests/pending/test_models.py
+ --ignore=tests/pending/test_optimizer_amsgrad.py
+ --ignore=tests/pending/test_logging_hyperdash.py
+ --ignore=tests/pending/test_yapf_format.py
+ --ignore=tests/pending/test_layers_normalization.py
+ --ignore=tests/pending/test_utils_predict.py
+ --ignore=tests/pending/test_layers_flow_control.py
+ --ignore=tests/performance_test/vgg/tl2-autograph.py
+ --ignore=tests/performance_test/vgg/tf2-eager.py
+ --ignore=tests/performance_test/vgg/exp_config.py
+ --ignore=tests/performance_test/vgg/tl2-eager.py
+ --ignore=tests/performance_test/vgg/tf2-autograph.py
+ --ignore=tests/performance_test/vgg/keras_test.py
+ --ignore=tests/performance_test/vgg/pytorch_test.py
+
+[flake8]
+max-line-length = 120
+ignore =
+ D301
+ E221 # Space before equal sign
+ E251 # Space after equal sign
+exclude =
+ .git,
+ venv,
+ __pycache__,
+ .pytest_cache,
+ tensorlayer.egg-info,
+ build,
+ dist,
+ img
+
+[yapf]
+based_on_style=google
+
+# The number of columns to use for indentation.
+indent_width = 4
+
+# The column limit.
+column_limit=120
+
+# Place each dictionary entry onto its own line.
+each_dict_entry_on_separate_line = True
+
+# Put closing brackets on a separate line, dedented, if the bracketed
+# expression can't fit in a single line. Applies to all kinds of brackets,
+# including function definitions and calls. For example:
+#
+# config = {
+# 'key1': 'value1',
+# 'key2': 'value2',
+# } # <--- this bracket is dedented and on a separate line
+#
+# time_series = self.remote_client.query_entity_counters(
+# entity='dev3246.region1',
+# key='dns.query_latency_tcp',
+# transform=Transformation.AVERAGE(window=timedelta(seconds=60)),
+# start_ts=now()-timedelta(days=3),
+# end_ts=now(),
+# ) # <--- this bracket is dedented and on a separate line
+dedent_closing_brackets=True
+
+# Do not split consecutive brackets. Only relevant when DEDENT_CLOSING_BRACKETS is set
+coalesce_brackets = False
+
+# Align closing bracket with visual indentation.
+align_closing_bracket_with_visual_indent = False
+
+# Split named assignments onto individual lines.
+split_before_named_assigns = False
+
+# If an argument / parameter list is going to be split, then split before the first argument.
+split_before_first_argument = True
+
+# Split before arguments if the argument list is terminated by a comma.
+split_arguments_when_comma_terminated = False
+
+# Insert a space between the ending comma and closing bracket of a list, etc.
+space_between_ending_comma_and_closing_bracket = True
+
+# Join short lines into one line. E.g., single line if statements.
+join_multiple_lines = True
+
+# Do not include spaces around selected binary operators.
+# Example: 1 + 2 * 3 - 4 / 5 => 1 + 2*3 - 4/5
+no_spaces_around_selected_binary_operators = True
+
+# Allow lambdas to be formatted on more than one line.
+allow_multiline_lambdas = True
+
+SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT = 10
+SPLIT_PENALTY_AFTER_OPENING_BRACKET = 500
diff --git a/setup.travis_doc.cfg b/setup.travis_doc.cfg
new file mode 100644
index 0000000..55267cf
--- /dev/null
+++ b/setup.travis_doc.cfg
@@ -0,0 +1,81 @@
+[tool:pytest]
+testpaths = tests/
+python_files=*test_documentation*
+ *test_yapf_format*
+
+[flake8]
+max-line-length = 120
+ignore =
+ D301
+ E221 # Space before equal sign
+ E251 # Space after equal sign
+exclude =
+ .git,
+ venv,
+ __pycache__,
+ .pytest_cache,
+ tensorlayer.egg-info,
+ build,
+ dist,
+ img
+
+[yapf]
+based_on_style=google
+
+# The number of columns to use for indentation.
+indent_width = 4
+
+# The column limit.
+column_limit=120
+
+# Place each dictionary entry onto its own line.
+each_dict_entry_on_separate_line = True
+
+# Put closing brackets on a separate line, dedented, if the bracketed
+# expression can't fit in a single line. Applies to all kinds of brackets,
+# including function definitions and calls. For example:
+#
+# config = {
+# 'key1': 'value1',
+# 'key2': 'value2',
+# } # <--- this bracket is dedented and on a separate line
+#
+# time_series = self.remote_client.query_entity_counters(
+# entity='dev3246.region1',
+# key='dns.query_latency_tcp',
+# transform=Transformation.AVERAGE(window=timedelta(seconds=60)),
+# start_ts=now()-timedelta(days=3),
+# end_ts=now(),
+# ) # <--- this bracket is dedented and on a separate line
+dedent_closing_brackets=True
+
+# Do not split consecutive brackets. Only relevant when DEDENT_CLOSING_BRACKETS is set
+coalesce_brackets = False
+
+# Align closing bracket with visual indentation.
+align_closing_bracket_with_visual_indent = False
+
+# Split named assignments onto individual lines.
+split_before_named_assigns = False
+
+# If an argument / parameter list is going to be split, then split before the first argument.
+split_before_first_argument = True
+
+# Split before arguments if the argument list is terminated by a comma.
+split_arguments_when_comma_terminated = False
+
+# Insert a space between the ending comma and closing bracket of a list, etc.
+space_between_ending_comma_and_closing_bracket = True
+
+# Join short lines into one line. E.g., single line if statements.
+join_multiple_lines = True
+
+# Do not include spaces around selected binary operators.
+# Example: 1 + 2 * 3 - 4 / 5 => 1 + 2*3 - 4/5
+no_spaces_around_selected_binary_operators = True
+
+# Allow lambdas to be formatted on more than one line.
+allow_multiline_lambdas = True
+
+SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT = 10
+SPLIT_PENALTY_AFTER_OPENING_BRACKET = 500
\ No newline at end of file
diff --git a/tensorlayer/__init__.py b/tensorlayer/__init__.py
new file mode 100644
index 0000000..b111a3e
--- /dev/null
+++ b/tensorlayer/__init__.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Deep learning and Reinforcement learning library for Researchers and Engineers"""
+
+# import backend
+from .backend import *
+# from .backend import ops
+# import dataflow
+# from .dataflow import *
+
+import os
+from distutils.version import LooseVersion
+
+from tensorlayer.package_info import (
+ VERSION, __contact_emails__, __contact_names__, __description__, __download_url__, __homepage__, __keywords__,
+ __license__, __package_name__, __repository_url__, __shortversion__, __version__
+)
+
+if 'TENSORLAYER_PACKAGE_BUILDING' not in os.environ:
+
+ try:
+ import tensorflow
+ except Exception as e:
+ raise ImportError(
+ "Tensorflow is not installed, please install it with the one of the following commands:\n"
+ " - `pip install --upgrade tensorflow`\n"
+ " - `pip install --upgrade tensorflow-gpu`"
+ )
+
+ if ("SPHINXBUILD" not in os.environ and "READTHEDOCS" not in os.environ and
+ LooseVersion(tensorflow.__version__) < LooseVersion("2.0.0")):
+ raise RuntimeError(
+ "TensorLayer does not support Tensorflow version older than 2.0.0.\n"
+ "Please update Tensorflow with:\n"
+ " - `pip install --upgrade tensorflow`\n"
+ " - `pip install --upgrade tensorflow-gpu`"
+ )
+
+ from tensorlayer import array_ops
+ from tensorlayer import cost
+ from tensorlayer import decorators
+ from tensorlayer import files
+ from tensorlayer import initializers
+ from tensorlayer import iterate
+ from tensorlayer import layers
+ from tensorlayer import lazy_imports
+ from tensorlayer import logging
+ from tensorlayer import models
+ from tensorlayer import optimizers
+ from tensorlayer import rein
+ from tensorlayer import utils
+ from tensorlayer import dataflow
+
+ from tensorlayer.lazy_imports import LazyImport
+
+ # Lazy Imports
+ db = LazyImport("tensorlayer.db")
+ distributed = LazyImport("tensorlayer.distributed")
+ nlp = LazyImport("tensorlayer.nlp")
+ prepro = LazyImport("tensorlayer.prepro")
+ utils = LazyImport("tensorlayer.utils")
+ visualize = LazyImport("tensorlayer.visualize")
+
+ # alias
+ vis = visualize
+
+ alphas = array_ops.alphas
+ alphas_like = array_ops.alphas_like
+
+ # global vars
+ global_flag = {}
+ global_dict = {}
diff --git a/tensorlayer/array_ops.py b/tensorlayer/array_ops.py
new file mode 100644
index 0000000..f09e833
--- /dev/null
+++ b/tensorlayer/array_ops.py
@@ -0,0 +1,110 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""A file containing functions related to array manipulation."""
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op, dtypes, ops, tensor_shape
+from tensorflow.python.framework.constant_op import constant
+from tensorflow.python.framework.ops import convert_to_tensor
+from tensorflow.python.ops.array_ops import shape_internal
+from tensorflow.python.ops.gen_array_ops import fill, reshape
+
+__all__ = ['alphas', 'alphas_like']
+
+
+def alphas(shape, alpha_value, name=None):
+ """Creates a tensor with all elements set to `alpha_value`.
+ This operation returns a tensor of type `dtype` with shape `shape` and all
+ elements set to alpha.
+
+ Parameters
+ ----------
+ shape: A list of integers, a tuple of integers, or a 1-D `Tensor` of type `int32`.
+ The shape of the desired tensor
+ alpha_value: `float32`, `float64`, `int8`, `uint8`, `int16`, `uint16`, int32`, `int64`
+ The value used to fill the resulting `Tensor`.
+ name: str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A `Tensor` with all elements set to alpha.
+
+ Examples
+ --------
+ >>> tl.alphas([2, 3], tf.int32) # [[alpha, alpha, alpha], [alpha, alpha, alpha]]
+ """
+ with ops.name_scope(name, "alphas", [shape]) as name:
+
+ alpha_tensor = convert_to_tensor(alpha_value)
+ alpha_dtype = dtypes.as_dtype(alpha_tensor.dtype).base_dtype
+
+ if not isinstance(shape, ops.Tensor):
+ try:
+ shape = constant_op._tensor_shape_tensor_conversion_function(tensor_shape.TensorShape(shape))
+ except (TypeError, ValueError):
+ shape = ops.convert_to_tensor(shape, dtype=dtypes.int32)
+
+ if not shape._shape_tuple():
+ shape = reshape(shape, [-1]) # Ensure it's a vector
+
+ try:
+ output = constant(alpha_value, shape=shape, dtype=alpha_dtype, name=name)
+
+ except (TypeError, ValueError):
+ output = fill(shape, constant(alpha_value, dtype=alpha_dtype), name=name)
+
+ if output.dtype.base_dtype != alpha_dtype:
+ raise AssertionError("Dtypes do not corresponds: %s and %s" % (output.dtype.base_dtype, alpha_dtype))
+
+ return output
+
+
+def alphas_like(tensor, alpha_value, name=None, optimize=True):
+ """Creates a tensor with all elements set to `alpha_value`.
+ Given a single tensor (`tensor`), this operation returns a tensor of the same
+ type and shape as `tensor` with all elements set to `alpha_value`.
+
+ Parameters
+ ----------
+ tensor: tf.Tensor
+ The Tensorflow Tensor that will be used as a template.
+ alpha_value: `float32`, `float64`, `int8`, `uint8`, `int16`, `uint16`, int32`, `int64`
+ The value used to fill the resulting `Tensor`.
+ name: str
+ A name for the operation (optional).
+ optimize: bool
+ if true, attempt to statically determine the shape of 'tensor' and encode it as a constant.
+
+ Returns
+ -------
+ A `Tensor` with all elements set to `alpha_value`.
+
+ Examples
+ --------
+ >>> tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
+ >>> tl.alphas_like(tensor, 0.5) # [[0.5, 0.5, 0.5], [0.5, 0.5, 0.5]]
+ """
+ with ops.name_scope(name, "alphas_like", [tensor]) as name:
+ tensor = ops.convert_to_tensor(tensor, name="tensor")
+
+ if context.in_eager_mode(): # and dtype is not None and dtype != tensor.dtype:
+ ret = alphas(shape_internal(tensor, optimize=optimize), alpha_value=alpha_value, name=name)
+
+ else: # if context.in_graph_mode():
+
+ # For now, variant types must be created via zeros_like; as we need to
+ # pass the input variant object to the proper zeros callback.
+
+ if (optimize and tensor.shape.is_fully_defined()):
+ # We can produce a zeros tensor independent of the value of 'tensor',
+ # since the shape is known statically.
+ ret = alphas(tensor.shape, alpha_value=alpha_value, name=name)
+
+ # elif dtype is not None and dtype != tensor.dtype and dtype != dtypes.variant:
+ else:
+ ret = alphas(shape_internal(tensor, optimize=optimize), alpha_value=alpha_value, name=name)
+
+ ret.set_shape(tensor.get_shape())
+
+ return ret
diff --git a/tensorlayer/backend/__init__.py b/tensorlayer/backend/__init__.py
new file mode 100644
index 0000000..01e5c83
--- /dev/null
+++ b/tensorlayer/backend/__init__.py
@@ -0,0 +1,6 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+# load ops
+from .ops import *
+from tensorlayer.backend import ops
\ No newline at end of file
diff --git a/tensorlayer/backend/ops/__init__.py b/tensorlayer/backend/ops/__init__.py
new file mode 100644
index 0000000..5b8a61b
--- /dev/null
+++ b/tensorlayer/backend/ops/__init__.py
@@ -0,0 +1,126 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+# load nn ops
+from .load_backend import padding_format
+from .load_backend import preprocess_1d_format
+from .load_backend import preprocess_2d_format
+from .load_backend import preprocess_3d_format
+from .load_backend import nchw_to_nhwc
+from .load_backend import nhwc_to_nchw
+from .load_backend import relu
+from .load_backend import relu6
+from .load_backend import leaky_relu
+from .load_backend import softplus
+from .load_backend import tanh
+from .load_backend import sigmoid
+from .load_backend import softmax
+from .load_backend import bias_add
+from .load_backend import conv1d
+from .load_backend import conv2d
+from .load_backend import conv3d
+from .load_backend import lrn
+from .load_backend import moments
+from .load_backend import max_pool
+from .load_backend import avg_pool
+from .load_backend import max_pool3d
+from .load_backend import avg_pool3d
+from .load_backend import pool
+from .load_backend import depthwise_conv2d
+from .load_backend import Conv1d_transpose
+from .load_backend import Conv2d_transpose
+from .load_backend import Conv3d_transpose
+
+from .load_backend import ReLU
+from .load_backend import ReLU6
+from .load_backend import LeakyReLU
+from .load_backend import Softplus
+from .load_backend import Tanh
+from .load_backend import Sigmoid
+from .load_backend import Softmax
+from .load_backend import Conv1D
+from .load_backend import Conv2D
+from .load_backend import Conv3D
+from .load_backend import BiasAdd
+from .load_backend import MaxPool
+from .load_backend import AvgPool
+from .load_backend import Dropout
+from .load_backend import BatchNorm
+from .load_backend import DepthwiseConv2d
+
+# load ops
+from .load_backend import Variable
+from .load_backend import matmul
+from .load_backend import add
+from .load_backend import dtypes
+from .load_backend import minimum
+from .load_backend import reshape
+from .load_backend import concat
+from .load_backend import convert_to_tensor
+from .load_backend import sqrt
+from .load_backend import reduce_mean
+from .load_backend import reduce_min
+from .load_backend import reduce_max
+from .load_backend import pad
+from .load_backend import stack
+from .load_backend import meshgrid
+from .load_backend import range
+from .load_backend import expand_dims
+from .load_backend import tile
+from .load_backend import cast
+from .load_backend import transpose
+from .load_backend import gather_nd
+from .load_backend import clip_by_value
+from .load_backend import split
+from .load_backend import get_tensor_shape
+from .load_backend import set_context
+from .load_backend import resize
+from .load_backend import floor
+from .load_backend import gather
+from .load_backend import linspace
+from .load_backend import slice
+from .load_backend import add_n
+from .load_backend import ceil
+from .load_backend import multiply
+from .load_backend import divide
+from .load_backend import identity
+
+# dtype
+from .load_backend import (DType, float16, float32, float64, int8, int16, int32, int64, uint8, uint16, uint32, uint64)
+# initlizers
+from .load_backend import (zeros, ones, constant, random_uniform, random_normal, truncated_normal, he_normal)
+# backend
+from .load_backend import BACKEND
+from .load_backend import BACKEND_VERSION
+
+from .load_backend import Reshape
+from .load_backend import ReduceSum
+from .load_backend import ReduceMax
+from .load_backend import ReduceMean
+from .load_backend import OneHot
+from .load_backend import L2Normalize
+from .load_backend import EmbeddingLookup
+from .load_backend import NCELoss
+from .load_backend import Not_equal
+from .load_backend import Cast
+from .load_backend import ExpandDims
+from .load_backend import Count_nonzero
+from .load_backend import FlattenReshape
+from .load_backend import Transpose
+from .load_backend import MatMul
+from .load_backend import Tile
+from .load_backend import Concat
+from .load_backend import ZeroPadding1D
+from .load_backend import ZeroPadding2D
+from .load_backend import ZeroPadding3D
+from .load_backend import Stack
+from .load_backend import Unstack
+from .load_backend import Sign
+from .load_backend import Resize
+from .load_backend import Pad
+from .load_backend import Minimum
+from .load_backend import Maximum
+from .load_backend import Meshgrid
+from .load_backend import BatchToSpace
+from .load_backend import DepthToSpace
+
diff --git a/tensorlayer/backend/ops/dragon_backend.py b/tensorlayer/backend/ops/dragon_backend.py
new file mode 100644
index 0000000..37e6e5a
--- /dev/null
+++ b/tensorlayer/backend/ops/dragon_backend.py
@@ -0,0 +1,1039 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, division, print_function
+
+import numpy as np
+import dragon as D
+
+from dragon.core.eager import context
+from dragon.core.ops import init_ops
+from dragon.core.ops import vision_ops
+
+_dtypeDict = ['float16', 'float32', 'float64', 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64']
+# TODO NotImplemented
+DType = None
+float16 = 'float16'
+float32 = 'float32'
+float64 = 'float64'
+int8 = 'int8'
+int16 = 'int16'
+int32 = 'int32'
+int64 = 'int64'
+uint8 = 'uint8'
+uint16 = 'uint16'
+uint32 = 'uint32'
+uint64 = 'uint64'
+
+# isinstance input output
+# TODO NotImplemented
+# TensorLike = None
+
+
+def _getter(init_fn, **kwargs):
+ """Return an named eager tensor."""
+ with context.eager_mode():
+ value = init_fn(**kwargs)
+ value._name = kwargs.get('name', value.id)
+ return value
+
+
+def set_context(**kwargs):
+ raise Exception("Using Dragon backend,You don't need to set context")
+
+
+def get_tensor_shape(x):
+ return x.shape
+
+
+# initializers
+def zeros(shape, dtype='float32'):
+ """
+ Creates a tensor with all elements set to zero.
+
+ Parameters
+ ----------
+ shape : A list of integers
+ a tuple of integers, or a 1-D Tensor of type int32.
+ dtype : tensor
+ The DType of an element in the resulting Tensor
+
+ Returns
+ -------
+ A Tensor with all elements set to zero.
+
+ """
+ return _getter(
+ init_ops.fill,
+ value=0,
+ shape=shape,
+ dtype=dtype,
+ )
+
+
+def ones(shape, dtype='float32'):
+ """
+ Creates a tensor with all elements set to ones.
+
+ Parameters
+ ----------
+ shape : A list of integers
+ a tuple of integers, or a 1-D Tensor of type int32.
+ dtype : tensor
+ The DType of an element in the resulting Tensor
+
+ Returns
+ -------
+ A Tensor with all elements set to zero.
+
+ """
+ return _getter(
+ init_ops.fill,
+ value=1,
+ shape=shape,
+ dtype=dtype,
+ )
+
+
+def constant(value, shape, dtype='float32'):
+ """
+ Creates a constant tensor from a tensor-like object.
+
+ Parameters
+ ----------
+ value : list
+ A constant value (or list) of output type dtype.
+ dtype : tensor
+ The type of the elements of the resulting tensor.
+ shape : tuple
+ Optional dimensions of resulting tensor.
+
+ Returns
+ -------
+ A Constant Tensor.
+
+ """
+ # shape = shape[::-1]
+ return _getter(
+ init_ops.fill,
+ value=value,
+ shape=shape,
+ dtype=dtype,
+ )
+
+
+def random_uniform(shape, minval=0, maxval=None, dtype='float32', seed=None):
+ """
+ Outputs random values from a uniform distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ minval : int
+ The lower bound on the range of random values to generate (inclusive). Defaults to 0.
+ maxval : int
+ The upper bound on the range of random values to generate (exclusive). Defaults to 1 if dtype is floating point.
+ dtype : tensor
+ The type of the output: float16, float32, float64, int32, or int64.
+ seed : int
+ Used in combination with dragon.random.set_seed to create a reproducible sequence of tensors across multiple calls.
+ Returns
+ -------
+ A tensor of the specified shape filled with random uniform values.
+
+ """
+ return _getter(init_ops.random_uniform, low=minval, high=maxval, shape=shape, dtype=dtype)
+
+
+def random_normal(shape, mean=0.0, stddev=1.0, dtype='float32', seed=None):
+ """
+ Outputs random values from a normal distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ dtype : tensor
+ The type of the output.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns
+ -------
+ A tensor of the specified shape filled with random normal values.
+
+ """
+ return _getter(
+ init_ops.random_normal,
+ mean=mean,
+ std=stddev,
+ shape=shape,
+ dtype=dtype,
+ )
+
+
+def truncated_normal(shape, mean=0.0, stddev=1.0, dtype='float32', seed=None):
+ """
+ Outputs random values from a truncated normal distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ dtype : tensor
+ The type of the output.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns
+ -------
+ A tensor of the specified shape filled with random truncated normal values.
+
+ """
+ return _getter(
+ init_ops.truncated_normal,
+ mean=mean,
+ std=stddev,
+ shape=shape,
+ dtype=dtype,
+ )
+
+
+def he_normal(shape, dtype, seed=None):
+ """
+ He normal initializer.
+
+ Parameters
+ ----------
+ seed : A Python integer.
+ Used to seed the random generator.
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ dtype : tensor
+ The type of the output.
+
+ Returns
+ -------
+ A tensor of the specified shape filled with he normal values.
+ """
+ # shape = shape[::-1]
+ raise NotImplementedError("He_Normal is not implemented")
+
+
+def Variable(initial_value, name, trainable=None):
+ """
+ Creates a new variable with value initial_value.
+
+ Parameters
+ ----------
+ initial_value : tensor
+ A Tensor, or Python object convertible to a Tensor
+ name : str
+ Optional name for the variable. Defaults to 'Variable' and gets uniquified automatically.
+ Returns
+ -------
+ Variable
+ """
+ return D.Tensor(name=name, shape=initial_value)
+
+
+class MatMul(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, a, b):
+ inputs = [a, b]
+ return D.math.matmul(inputs)
+
+
+def matmul(a, b):
+ """
+ Multiplies matrix a by matrix b, producing a * b.
+
+ Parameters
+ ----------
+ a : tensor
+ type float16, float32, float64, int32, complex64, complex128 and rank > 1.
+ b : tensor
+ with same type and rank as a.
+
+ Returns
+ -------
+ A Tensor of the same type as a and b
+ """
+ inputs = [a, b]
+ return D.math.matmul(inputs)
+
+
+def add(value, bias):
+ """
+ Returns x + y element-wise.
+
+ Parameters
+ ----------
+ value : tensor.
+ Must be one of the following types: bfloat16, half, float32, float64,
+ uint8, int8, int16, int32, int64, complex64, complex128, string.
+ bias : tensor
+ Must have the same type as a
+ name : str
+ A name for the operation
+
+ Returns
+ -------
+ A Tensor. Has the same type as a.
+ """
+
+ inputs = [value, bias]
+ return D.math.add(inputs)
+
+
+def dtypes(dt):
+ """
+ Data dtypes.
+
+ Parameters
+ ----------
+ dt : string
+ It could be 'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16',
+ 'int32', 'int64', 'float16', 'float32', 'float64', 'DType'.
+
+ Returns
+ -------
+ Data dtypes
+ """
+ if dt not in _dtypeDict:
+ raise Exception("Unsupported dtype: {}".format(dt))
+ return dt
+
+
+def minimum(x, y):
+ """
+ Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+
+ Parameters
+ ----------
+ x : tensor.
+ Must be one of the following types: bfloat16, half, float32, float64, int32, int64.
+ y : A Tensor.
+ Must have the same type as x.
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor. Has the same type as x
+ """
+ inputs = [x, y]
+ return D.math.minimum(inputs)
+
+
+class FlattenReshape(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, inputs):
+ dim = 1
+ for d in get_tensor_shape(inputs)[1:]:
+ dim *= d
+ return D.reshape(inputs, [-1, dim])
+
+
+class Reshape(object):
+
+ def __init__(self, shape):
+ self.shape = shape
+
+ def __call__(self, tensor):
+ return D.reshape(tensor, shape=self.shape)
+
+
+def reshape(tensor, shape):
+ """
+ Reshapes a tensor.
+
+ Parameters
+ ----------
+ tensor : tensor
+ A Tensor.
+ shape : tensor
+ Defines the shape of the output tensor.
+ Returns
+ -------
+ A Tensor. Has the same type as tensor
+ """
+ return D.reshape(tensor, shape=shape)
+
+
+class Concat(object):
+
+ def __init__(self, axis):
+ super(Concat, self).__init__()
+ self.axis = axis
+
+ def __call__(self, values):
+ return D.concat(values=values, axis=self.axis)
+
+
+def concat(values, axis):
+ """
+ Concatenates tensors along one dimension.
+
+ Parameters
+ ----------
+ values : list
+ A list of Tensor objects or a single Tensor
+ axis : int
+ 0-D int32 Tensor. Dimension along which to concatenate
+ Returns
+ -------
+ A Tensor resulting from concatenation of the input tensors.
+ """
+ return D.concat(values, axis=axis)
+
+
+def convert_to_tensor(value, dtype=None):
+ """
+ Converts the given value to a Tensor.
+
+ Parameters
+ ----------
+ value : object
+ An object whose type has a registered Tensor conversion function.
+ dtype : optional
+ Optional element type for the returned tensor. If missing, the type is inferred from the type of value.
+
+ Returns
+ -------
+ A Tensor based on value.
+ """
+ return D.Tensor.convert_to(value, dtype)
+
+
+def sqrt(x):
+ """
+ Computes square root of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x.
+ """
+ return D.math.sqrt(x)
+
+
+class ReduceSum(object):
+
+ def __init__(self, axis):
+ pass
+
+ def construct(self, input):
+ pass
+
+
+class ReduceMean(object):
+
+ def __init__(self, axis):
+ if axis == [1, 2]:
+ self.data_format = 'NHWC'
+ elif axis == [2, 3]:
+ self.data_format = 'NCHW'
+ else:
+ raise ("`data_format` should have one of the following values: [`channels_last`, `channels_first`]")
+
+ def __call__(self, inputs):
+ return vision_ops.pool2d(
+ inputs,
+ kernel_shape=1,
+ strides=1,
+ pads=0,
+ mode='AVG',
+ global_pooling=True,
+ data_format=self.data_format,
+ )
+
+
+def reduce_mean(input_tensor, axis=None):
+ """
+ Computes the mean of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ return D.mean(input_tensor, axes=axis)
+
+
+class ReduceMax(object):
+
+ def __init__(self, axis):
+ if axis == [1, 2]:
+ self.data_format = 'NHWC'
+ elif axis == [2, 3]:
+ self.data_format = 'NCHW'
+ else:
+ raise ("`data_format` should have one of the following values: [`channels_last`, `channels_first`]")
+
+ def __call__(self, inputs):
+ return vision_ops.pool2d(
+ inputs, kernel_shape=1, strides=1, pads=0, mode='MAX', global_pooling=True, data_format=self.data_format
+ )
+
+
+def reduce_max(input_tensor, axis=None):
+ """
+ Computes the maximum of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have real numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ return D.max(input_tensor, axis)
+
+
+def reduce_min(input_tensor, axis=None):
+ """
+ Computes the minimum of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have real numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+ return D.min(input_tensor, axis)
+
+class Pad(object):
+ def __init__(self, paddings, mode="REFLECT"):
+ if mode not in ['CONSTANT', 'REFLECT', 'SYMMETRIC']:
+ raise Exception("Unsupported mode: {}".format(mode))
+ if mode == 'SYMMETRIC':
+ mode = 'EDGE'
+ self.paddings = paddings
+ self.mode = mode
+
+ def __call__(self, x):
+ outputs = D.pad(x, pads=self.paddings, mode=self.mode, value=0)
+ return outputs
+
+def pad(tensor, paddings, mode='CONSTANT', constant_values=0):
+ """
+ Pads a tensor.
+
+ Parameters
+ ----------
+ tensor : tensor
+ A Tensor.
+ paddings : tuple
+ A tuple of type int32.
+ mode : str
+ One of "CONSTANT", "REFLECT", or "SYMMETRIC" (case-insensitive)
+ constant_values : int
+ In "CONSTANT" mode, the scalar pad value to use. Must be same type as tensor.
+
+ Returns
+ -------
+ A Tensor. Has the same type as tensor.
+ """
+ if mode not in ['CONSTANT', 'REFLECT', 'SYMMETRIC']:
+ raise Exception("Unsupported mode: {}".format(mode))
+ if mode == 'SYMMETRIC':
+ mode = 'EDGE'
+ outputs = D.pad(tensor, pads=paddings, mode=mode, value=constant_values)
+ return outputs
+
+
+class Unstack(object):
+
+ def __init__(self, axis, num=None):
+ self.axis = axis
+ self.num = num
+
+ def __call__(self, values):
+ raise NotImplementedError
+
+
+class Stack(object):
+
+ def __init__(self, axis):
+ self.axis = axis
+
+ def __call__(self, values):
+ return D.stack(values, axis=self.axis)
+
+
+def stack(values, axis=0):
+ """
+ Stacks a list of rank-R tensors into one rank-(R+1) tensor.
+
+ Parameters
+ ----------
+ values : list
+ A list of Tensor objects with the same shape and type.
+ axis : int
+ An int. The axis to stack along. Defaults to the first dimension.
+ Negative values wrap around, so the valid range is [-(R+1), R+1).
+
+ Returns
+ -------
+ A stacked Tensor with the same type as values.
+ """
+ return D.stack(values, axis=axis)
+
+
+class Meshgrid(object):
+ def __init__(self, indexing='xy'):
+ super(Meshgrid, self).__init__()
+ self.index = indexing
+
+ def __call__(self, inputs):
+ pass
+
+
+def meshgrid(x, y):
+ """
+ Broadcasts parameters for evaluation on an N-D grid.
+
+ Parameters
+ ----------
+ x : tensor
+ Tensors with rank 1.
+ y : tensor
+ Tensors with rank 1.
+
+ Returns
+ -------
+ A list of N Tensors with rank N.
+ """
+
+ pass
+
+
+def range(start, limit=None, delta=1, dtype=None):
+ """
+ Creates a sequence of numbers.
+
+ Parameters
+ ----------
+ start : tensor
+ A 0-D Tensor (scalar). Acts as first entry in the range if limit is not None;
+ otherwise, acts as range limit and first entry defaults to 0.
+ limit : tensor
+ A 0-D Tensor (scalar). Upper limit of sequence, exclusive. If None,
+ defaults to the value of start while the first entry of the range defaults to 0.
+ delta : tensor
+ A 0-D Tensor (scalar). Number that increments start. Defaults to 1.
+ dtype : type
+ The type of the elements of the resulting tensor.
+
+ Returns
+ -------
+ An 1-D Tensor of type dtype.
+ """
+ if dtype is None:
+ dtype = 'int32'
+ if limit is None:
+ outputs = D.arange(start=0, stop=start, step=delta, dtype=dtype)
+ else:
+ outputs = D.arange(start, stop=limit, step=delta, dtype=dtype)
+ return outputs
+
+
+class ExpandDims(object):
+
+ def __init__(self, axis):
+ pass
+
+ def construct(self, input):
+ pass
+
+
+def expand_dims(input, axis):
+ """
+ Inserts a dimension of 1 into a tensor's shape.
+
+ Parameters
+ ----------
+ input : tensor
+ A Tensor.
+ axis : int
+ 0-D (scalar). Specifies the dimension index at which to expand the shape of input.
+ Must be in the range [-rank(input) - 1, rank(input)].
+
+ Returns
+ -------
+ A Tensor with the same data as input, but its shape has an additional dimension of size 1 added.
+ """
+
+ return D.expand_dims(input, axis=axis)
+
+
+class Tile(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, input, multiples):
+ return D.tile(input, multiples)
+
+
+def tile(input, multiples):
+ """
+ Constructs a tensor by tiling a given tensor.
+
+ Parameters
+ ----------
+ input : tensor
+ A Tensor. 1-D or higher.
+ multiples : tensor
+ Must be one of the following types: int32, int64. 1-D.
+ Length must be the same as the number of dimensions in input
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ return D.tile(input, multiples)
+
+
+class Cast(object):
+
+ def __init__(self, dtype):
+ pass
+
+ def __call__(self, input):
+ pass
+
+
+def cast(x, dtype):
+ """
+ Casts a tensor to a new type.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor or SparseTensor or IndexedSlices of numeric type.
+ It could be uint8, uint16, uint32, uint64, int8, int16, int32, int64, float16, float32, float64.
+ dtype : dtpye
+ The destination type. The list of supported dtypes is the same as x
+
+ Returns
+ -------
+ A Tensor or SparseTensor or IndexedSlices with same shape as x and same type as dtype.
+ """
+ return D.cast(x, dtype=dtype)
+
+
+class Transpose(object):
+
+ def __init__(self, perm, conjugate=False):
+ self.perm = perm
+ if conjugate:
+ raise ("The conjugate Parameters not supported")
+
+ def __call__(self, a):
+ return D.transpose(a, self.perm)
+
+
+def transpose(a, perm=None, conjugate=False):
+ """
+ Transposes a.
+
+ Parameters
+ ----------
+ a : tensor
+ A Tensor.
+ perm : int
+ A permutation of the dimensions of a.
+ conjugate : bool
+ Setting it to True is mathematically equivalent to ms.math.conj(ms.transpose(input)).
+
+ Returns
+ -------
+ A transposed Tensor.
+ """
+
+ conjugate = conjugate
+ return D.transpose(a, perm=perm)
+
+
+def gather_nd(params, indices, batch_dims=0):
+ """
+ Gather slices from params into a Tensor with shape specified by indices.
+
+ Parameters
+ ----------
+ params : tensor
+ The tensor from which to gather values.
+ indices : tensor
+ Must be one of the following types: int32, int64. Index tensor.
+ batch_dims : int
+ An integer or a scalar 'Tensor'. The number of batch dimensions.
+
+ Returns
+ -------
+ A Tensor. Has the same type as params.
+ """
+
+ pass
+
+
+def clip_by_value(t, clip_value_min, clip_value_max):
+ """
+ Clips tensor values to a specified min and max.
+
+ Parameters
+ ----------
+ t : tensor
+ A Tensor or IndexedSlices
+ clip_value_min : tensor
+ A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by
+ clip_value_max : tensor
+ A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by
+
+ Returns
+ -------
+ A clipped Tensor or IndexedSlices.
+ """
+
+ pass
+
+
+def split(value, num_or_size_splits, axis=0, num=None):
+ """
+ Splits a tensor into sub tensors.
+
+ Parameters
+ ----------
+ value : tensor
+ The Tensor to split.
+ num_or_size_splits : list
+ Either an integer indicating the number of splits along split_dim or a 1-D integer Tensor or
+ Python list containing the sizes of each output tensor along split_dim.
+ axis : int
+ The dimension along which to split. Must be in the range [-rank(value), rank(value)). Defaults to 0.
+ num : int
+ used to specify the number of outputs when it cannot be inferred from the shape of size_splits.
+
+ Returns
+ -------
+ Tensor objects resulting from splitting value.
+ """
+ pass
+
+
+def floor(x):
+ return D.math.floor(x)
+
+
+def gather(params, indices):
+ return NotImplementedError
+
+
+def linspace(start, stop, num):
+ return D.linspace(start, stop, num)
+
+
+def slice(inputs, starts, sizes):
+ return D.slice(inputs, starts, sizes)
+
+
+def add_n(inputs):
+ return NotImplementedError
+
+
+class OneHot(object):
+
+ def __init__(self, axis=-1, depth=1, on_value=1.0, off_value=0.0, dtype='float32'):
+ self.depth = depth
+ self.dtype = dtype
+
+ def __call__(self, indices):
+ outputs = np.zeros(shape=(indices.shape[0], self.depth))
+ for i in np.arange(indices.shape[0]):
+ outputs[int(i)][int(indices[int(i)].get_value())] = 1
+ outputs = D.constant(outputs, dtype=self.dtype)
+ return outputs
+
+
+class L2Normalize(object):
+
+ def __init__(self, axis=None, epsilon=1e-12):
+ super(L2Normalize, self).__init__()
+ pass
+
+ def __call__(self, input, *args, **kwargs):
+ pass
+
+
+class EmbeddingLookup(object):
+
+ def __init__(self, max_norm=None):
+ self.max_norm = max_norm
+
+ def __call__(self, params, ids, *args, **kwargs):
+ pass
+
+
+class NCELoss(object):
+
+ def __init__(self, num_true=1, sampled_values=None, remove_accidental_hits=False):
+ super(NCELoss, self).__init__()
+
+ def __call__(self, weights, biases, labels, inputs, num_sampled, num_classes):
+ pass
+
+
+class Not_equal(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x, y):
+ pass
+
+
+class Count_nonzero(object):
+
+ def __init__(self, keepdims=None, dtype='int64'):
+ pass
+
+ def __call__(self, *args, **kwargs):
+ pass
+
+
+
+class Resize:
+
+ def __init__(self, scale, method, antialias=False, data_format='channels_last', ksize=None):
+ if method not in ['nearest', 'linear', 'bilinear']:
+ raise ('Current resize does not support this method.')
+ if method == 'bilinear':
+ method = 'linear'
+ self.method = method
+ self.antialias = antialias
+ self.scale = scale
+ if data_format != 'channel_last':
+ raise Exception("UpSampling2d resize_images only support channel_last")
+
+ def __call__(self, inputs):
+ output_size = (int(inputs.shape[1] * self.scale[0]), int(inputs.shape[2] * self.scale[1]))
+ outputs = D.vision.resize(inputs, sizes=output_size, mode=self.method, align_corners=self.antialias)
+ return outputs
+
+
+def resize(inputs, output_size, method, antialias):
+ if method not in ['nearest', 'linear', 'bilinear']:
+ raise ('Current resize does not support this method.')
+ if method == 'bilinear':
+ method = 'linear'
+ return D.vision.resize(inputs, sizes=output_size, mode=method, align_corners=antialias)
+
+
+class ZeroPadding1D(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, padding):
+ raise NotImplementedError
+
+
+class ZeroPadding2D(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, padding):
+ raise NotImplementedError
+
+
+class ZeroPadding3D(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, padding):
+ raise NotImplementedError
+
+
+class Sign(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return D.math.sign(x)
+
+def ceil(x):
+ raise NotImplementedError
+
+def multiply(x, y):
+ raise NotImplementedError
+
+def divide(x, y):
+ raise NotImplementedError
+
+def identity(x):
+ raise NotImplementedError
+
+class BatchToSpace(object):
+ def __init__(self, block_size, crops):
+ super(BatchToSpace, self).__init__()
+ pass
+
+ def __call__(self, input_x):
+ raise NotImplementedError
+
+
+class DepthToSpace(object):
+ def __init__(self, block_size, data_format='NHWC'):
+ pass
+
+ def __call__(self, input):
+ raise NotImplementedError
\ No newline at end of file
diff --git a/tensorlayer/backend/ops/dragon_nn.py b/tensorlayer/backend/ops/dragon_nn.py
new file mode 100644
index 0000000..e6b5105
--- /dev/null
+++ b/tensorlayer/backend/ops/dragon_nn.py
@@ -0,0 +1,910 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+import dragon as D
+from dragon.core.ops import vision_ops
+from dragon.core.ops import activation_ops
+
+
+def padding_format(padding):
+ """
+ Checks that the padding format correspond format.
+
+ Parameters
+ ----------
+ padding : str
+ Must be one of the following:"same", "SAME", "VALID", "valid"
+
+ Returns
+ -------
+ str "SAME" or "VALID"
+ """
+
+ if padding in ["SAME", "same"]:
+ padding = "SAME"
+ elif padding in ["VALID", "valid"]:
+ padding = "VALID"
+ elif padding == None:
+ padding = None
+ else:
+ raise Exception("Unsupported padding: " + str(padding))
+ return padding
+
+
+def preprocess_1d_format(data_format, padding):
+ """
+ Checks that the 1-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NWC","NCW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NWC" or "NCW" and "SAME" or "VALID"
+ """
+
+ if data_format in ["channels_last", "NWC"]:
+ data_format = "NWC"
+ elif data_format in ["channels_first", "NCW"]:
+ data_format = "NCW"
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def preprocess_2d_format(data_format, padding):
+ """
+ Checks that the 2-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NHWC","NCHW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NHWC" or "NCHW" and "SAME" or "VALID"
+ """
+
+ if data_format in ["channels_last", "NHWC", "nhwc"]:
+ data_format = "NHWC"
+ elif data_format in ["channels_first", "NCHW", "nchw"]:
+ data_format = "NCHW"
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def preprocess_3d_format(data_format, padding):
+ """
+ Checks that the 3-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NDHWC","NCDHW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NDHWC" or "NCDHW" and "SAME" or "VALID"
+ """
+
+ if data_format in ['channels_last', 'NDHWC']:
+ data_format = 'NDHWC'
+ elif data_format in ['channels_first', 'NCDHW']:
+ data_format = 'NCDHW'
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def nchw_to_nhwc(x):
+ """
+ Channels first to channels last
+
+ Parameters
+ ----------
+ x : tensor
+ channels first tensor data
+
+ Returns
+ -------
+ channels last tensor data
+ """
+
+ pass
+
+
+def nhwc_to_nchw(x):
+ """
+ Channles last to channels first
+
+ Parameters
+ ----------
+ x : tensor
+ channels last tensor data
+
+ Returns
+ -------
+ channels first tensor data
+ """
+
+ pass
+
+
+class ReLU(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return D.nn.relu(x)
+
+
+def relu(x):
+ """
+ Computes rectified linear: max(features, 0).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: float32, float64, int32, uint8, int16,
+ int8, int64, bfloat16, uint16, half, uint32, uint64, qint8.
+
+ Returns
+ -------
+ A Tensor. Has the same type as features.
+ """
+ return D.nn.relu(x)
+
+
+class ReLU6(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return D.nn.relu6(x)
+
+
+def relu6(x):
+ """
+ Computes Rectified Linear 6: min(max(features, 0), 6).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: float32, float64, int32, uint8, int16,
+ int8, int64, bfloat16, uint16, half, uint32, uint64, qint8.
+
+ Returns
+ -------
+ A Tensor with the same type as features.
+ """
+ return D.nn.relu6(x)
+
+
+class LeakyReLU(object):
+
+ def __init__(self, alpha=0.2):
+ self.alpha = alpha
+
+ def __call__(self, x):
+ return D.nn.leaky_relu(x, alpha=self.alpha)
+
+
+def leaky_relu(x):
+ """
+ Compute the Leaky ReLU activation function.
+
+ Parameters
+ ----------
+ x : tensor
+ representing preactivation values. Must be one of the following types:
+ float16, float32, float64, int32, int64.
+
+ Returns
+ -------
+ The activation value.
+ """
+
+ return D.nn.leaky_relu(x)
+
+
+class Softplus(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ raise NotImplementedError
+
+
+def softplus(x):
+ """
+ Computes softplus: log(exp(features) + 1).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: half, bfloat16, float32, float64.
+
+ Returns
+ -------
+ A Tensor. Has the same type as features.
+ """
+
+ raise NotImplementedError
+
+
+class Tanh(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return activation_ops.tanh(x)
+
+
+def tanh(x):
+ """
+ Computes hyperbolic tangent of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x.
+ """
+
+ return activation_ops.tanh(x)
+
+
+class Sigmoid(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return activation_ops.sigmoid(x)
+
+
+def sigmoid(x):
+ """
+ Computes sigmoid of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float16, float32, float64, complex64, or complex128.
+
+ Returns
+ -------
+ A Tensor with the same type as x.
+ """
+ return activation_ops.sigmoid(x)
+
+
+class Softmax(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return D.nn.softmax(x)
+
+
+def softmax(logits, axis=None):
+ """
+ Computes softmax activations.
+
+ Parameters
+ ----------
+ logits : tensor
+ Must be one of the following types: half, float32, float64.
+ axis : int
+ The dimension softmax would be performed on. The default is -1 which indicates the last dimension.
+
+ Returns
+ -------
+ A Tensor. Has the same type and shape as logits.
+ """
+ return D.nn.softmax(logits)
+
+
+class Dropout(object):
+
+ def __init__(self, keep, seed=1):
+ self.keep = 1 - keep
+ self.seed = seed
+
+ def __call__(self, inputs):
+ return D.nn.dropout(inputs, prob=self.keep)
+
+
+class BiasAdd(object):
+ """
+ Adds bias to value.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float, double, int64, int32, uint8, int16, int8, complex64, or complex128.
+ bias : tensor
+ Must be the same type as value unless value is a quantized type,
+ in which case a different quantized type may be used.
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ def __init__(self, data_format='NHWC'):
+ self.data_format = data_format
+
+ def __call__(self, x, bias):
+ inputs = [x, bias]
+ return vision_ops.bias_add(inputs, data_format=self.data_format)
+
+
+def bias_add(x, bias):
+ """
+ Adds bias to value.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float, double, int64, int32, uint8, int16, int8, complex64, or complex128.
+ bias : tensor
+ Must be the same type as value unless value is a quantized type,
+ in which case a different quantized type may be used.
+ data_format : A string.
+ 'N...C' and 'NC...' are supported.
+ name : str
+ A name for the operation (optional).
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+ inputs = [x, bias]
+ return vision_ops.bias_add(inputs, data_format='NHWC')
+
+
+class Conv1D(object):
+ pass
+ # raise NotImplementedError
+
+
+def conv1d(input, filters, stride, padding, data_format='NWC', dilations=None, name=None):
+ """
+ Computes a 1-D convolution given 3-D input and filter tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ A 3D Tensor. Must be of type float16, float32, or float64
+ filters : tensor
+ A 3D Tensor. Must have the same type as input.
+ stride : int of list
+ An int or list of ints that has length 1 or 3. The number of entries by which the filter is moved right at each step.
+ padding : string
+ 'SAME' or 'VALID'
+ data_format : string
+ An optional string from "NWC", "NCW". Defaults to "NWC", the data is stored in the order of
+ [batch, in_width, in_channels]. The "NCW" format stores data as [batch, in_channels, in_width].
+ dilations : int or list
+ An int or list of ints that has length 1 or 3 which defaults to 1.
+ The dilation factor for each dimension of input. If set to k > 1,
+ there will be k-1 skipped cells between each filter element on that dimension.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ A name for the operation (optional).
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ pass
+
+
+class Conv2D(object):
+
+ def __init__(self, strides, padding, data_format='NHWC', dilations=None, out_channel=None, k_size=None):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.ksize = k_size[0]
+ if self.data_format is 'NHWC':
+ self.dg_stride = strides[1]
+ self.dg_dilation = dilations[1]
+ elif self.data_format is 'NCHW':
+ self.dg_stride = strides[2]
+ self.dg_dilation = dilations[2]
+
+ def __call__(self, inputs, filters):
+ outputs = vision_ops.conv2d(
+ [inputs, filters],
+ kernel_shape=self.ksize,
+ strides=self.dg_stride,
+ padding=self.padding,
+ dilations=self.dg_dilation,
+ data_format=self.data_format,
+ )
+ return outputs
+
+
+def conv2d(input, filters, strides, padding, data_format='NCHW', dilations=None):
+ """
+ Computes a 2-D convolution given 4-D input and filters tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ Must be one of the following types: half, bfloat16, float32, float64. A 4-D tensor.
+ The dimension order is interpreted according to the value of data_format, see below for details.
+ filters : tensor
+ Must have the same type as input. A 4-D tensor of shape [filter_height, filter_width, in_channels, out_channels]
+ strides : int of list
+ The stride of the sliding window for each dimension of input. If a single value is given it is replicated in the H and W dimension.
+ By default the N and C dimensions are set to 1. The dimension order is determined by the value of data_format, see below for details.
+ padding : string
+ "SAME" or "VALID"
+ data_format : string
+ "NHWC", "NCHW". Defaults to "NCHW".
+ dilations : list or ints
+ list of ints that has length 1, 2 or 4, defaults to 1. The dilation factor for each dimension ofinput.
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ raise NotImplementedError
+
+
+class Conv3D(object):
+ pass
+ # raise NotImplementedError
+
+
+def conv3d(input, filters, strides, padding, data_format='NDHWC', dilations=None, name=None):
+ """
+ Computes a 3-D convolution given 5-D input and filters tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ Must be one of the following types: half, bfloat16, float32, float64.
+ Shape [batch, in_depth, in_height, in_width, in_channels].
+ filters : tensor
+ Must have the same type as input. Shape [filter_depth, filter_height, filter_width, in_channels, out_channels].
+ in_channels must match between input and filters.
+ strides : list of ints
+ A list of ints that has length >= 5. 1-D tensor of length 5.
+ The stride of the sliding window for each dimension of input.
+ Must have strides[0] = strides[4] = 1.
+ padding : string
+ A string from: "SAME", "VALID". The type of padding algorithm to use.
+ data_format : string
+ An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". The data format of the input and output data.
+ With the default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels].
+ Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width].
+ dilations : list of ints
+ Defaults to [1, 1, 1, 1, 1]. 1-D tensor of length 5. The dilation factor for each dimension of input.
+ If set to k > 1, there will be k-1 skipped cells between each filter element on that dimension.
+ The dimension order is determined by the value of data_format, see above for details.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ raise NotImplementedError
+
+
+def lrn(inputs, depth_radius, bias, alpha, beta):
+ """
+ Local Response Normalization.
+
+ Parameters
+ ----------
+ inputs : tensor
+ Must be one of the following types: half, bfloat16, float32. 4-D.
+ depth_radius : int
+ Defaults to 5. 0-D. Half-width of the 1-D normalization window.
+ bias : float
+ Defaults to 1. An offset (usually positive to avoid dividing by 0).
+ alpha : float
+ Defaults to 1. A scale factor, usually positive.
+ beta : float
+ Defaults to 0.5. An exponent.
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ pass
+
+
+def moments(x, axes, shift=None, keepdims=False):
+ """
+ Calculates the mean and variance of x.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor
+ axes : ints
+ Axes along which to compute mean and variance.
+ shift : int
+ Not used in the current implementation.
+ keepdims : bool
+ produce moments with the same dimensionality as the input.
+
+ Returns
+ -------
+ Two Tensor objects: mean and variance.
+ """
+
+ pass
+
+
+class MaxPool(object):
+
+ def __init__(self, ksize, strides, padding, data_format=None):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.ksize = ksize
+ self.strides = strides
+
+ def __call__(self, inputs):
+ return vision_ops.pool2d(
+ inputs,
+ kernel_shape=self.ksize,
+ strides=self.strides,
+ padding=self.padding,
+ mode='MAX',
+ global_pooling=False,
+ data_format=self.data_format,
+ )
+
+
+def max_pool(input, ksize, strides, padding, data_format=None):
+ """
+ Performs the max pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels] if data_format does not start
+ with "NC" (default), or [batch_size, num_channels] + input_spatial_shape if data_format starts with "NC".
+ Pooling happens over the spatial dimensions only.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The max pooled output tensor.
+ """
+ pass
+
+
+class AvgPool(object):
+
+ def __init__(self, ksize, strides, padding, data_format=None):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.filter_size = ksize
+ self.strides = strides
+
+ def __call__(self, inputs):
+ return vision_ops.pool2d(
+ inputs,
+ kernel_shape=self.filter_size,
+ strides=self.strides,
+ padding=self.padding,
+ mode='AVG',
+ global_pooling=False,
+ data_format=self.data_format,
+ )
+
+
+def avg_pool(input, ksize, strides, padding):
+ """
+ Performs the avg pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels]
+ if data_format does not start with "NC" (default), or [batch_size, num_channels] + input_spatial_shape
+ if data_format starts with "NC". Pooling happens over the spatial dimensions only.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The average pooled output tensor.
+ """
+ pass
+
+
+def max_pool3d(input, ksize, strides, padding, data_format=None, name=None):
+ """
+ Performs the max pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of the format specified by data_format.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ "NDHWC", "NCDHW". Defaults to "NDHWC". The data format of the input and output data.
+ With the default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels].
+ Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width].
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The max pooled output tensor.
+ """
+ pass
+
+
+def avg_pool3d(input, ksize, strides, padding, data_format=None, name=None):
+ """
+ Performs the average pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of shape [batch, height, width, channels] and type float32, float64, qint8, quint8, or qint32.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, 3 or 5. The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NDHWC' and 'NCDHW' are supported.
+ name : string
+ Optional name for the operation.
+
+ Returns
+ -------
+ A Tensor with the same type as value. The average pooled output tensor.
+ """
+ pass
+
+
+def pool(input, window_shape, pooling_type, strides=None, padding='VALID', data_format=None, dilations=None, name=None):
+ """
+ Performs an N-D pooling operation.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels]
+ if data_format does not start with "NC" (default), or [batch_size, num_channels] + input_spatial_shape
+ if data_format starts with "NC". Pooling happens over the spatial dimensions only.
+ window_shape : int
+ Sequence of N ints >= 1.
+ pooling_type : string
+ Specifies pooling operation, must be "AVG" or "MAX".
+ strides : ints
+ Sequence of N ints >= 1. Defaults to [1]*N. If any value of strides is > 1, then all values of dilation_rate must be 1.
+ padding : string
+ The padding algorithm, must be "SAME" or "VALID". Defaults to "SAME".
+ See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ Specifies whether the channel dimension of the input and output is the last dimension (default, or if data_format does not start with "NC"),
+ or the second dimension (if data_format starts with "NC").
+ For N=1, the valid values are "NWC" (default) and "NCW". For N=2, the valid values are "NHWC" (default) and "NCHW".
+ For N=3, the valid values are "NDHWC" (default) and "NCDHW".
+ dilations : list of ints
+ Dilation rate. List of N ints >= 1. Defaults to [1]*N. If any value of dilation_rate is > 1, then all values of strides must be 1.
+ name : string
+ Optional. Name of the op.
+
+ Returns
+ -------
+ Tensor of rank N+2, of shape [batch_size] + output_spatial_shape + [num_channels]
+ """
+ pass
+
+
+class DepthwiseConv2d(object):
+
+ def __init__(self, strides, padding, data_format=None, dilations=None, ksize=None, channel_multiplier=1):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.stride = strides
+ self.dilations = dilations
+
+ def __call__(self, input, filter):
+ raise NotImplementedError("Not implemented depthwiseconv2d")
+
+
+def depthwise_conv2d(input, filter, strides, padding, data_format=None, dilations=None, name=None):
+ """
+ Depthwise 2-D convolution.
+
+ Parameters
+ ----------
+ input : tensor
+ 4-D with shape according to data_format.
+ filter : tensor
+ 4-D with shape [filter_height, filter_width, in_channels, channel_multiplier].
+ strides : list
+ 1-D of size 4. The stride of the sliding window for each dimension of input.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ The data format for input. Either "NHWC" (default) or "NCHW".
+ dilations : list
+ 1-D of size 2. The dilation rate in which we sample input values across the height and width dimensions in atrous convolution.
+ If it is greater than 1, then all values of strides must be 1.
+ name : string
+ A name for this operation (optional).
+
+ Returns
+ -------
+ A 4-D Tensor with shape according to data_format.
+ E.g., for "NHWC" format, shape is [batch, out_height, out_width, in_channels * channel_multiplier].
+ """
+
+ pass
+
+
+def conv1d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NWC', dilations=None, name=None
+):
+ """
+ The transpose of conv1d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 3-D Tensor of type float and shape [batch, in_width, in_channels]
+ for NWC data format or [batch, in_channels, in_width] for NCW data format.
+ filters : tensor
+ A 3-D Tensor with the same type as value and shape [filter_width, output_channels, in_channels].
+ filter's in_channels dimension must match that of value.
+ output_shape : tensor
+ A 1-D Tensor, containing three elements, representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1 or 3. The number of entries by which the filter is moved right at each step.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NWC' and 'NCW' are supported.
+ dilations : list
+ An int or list of ints that has length 1 or 3 which defaults to 1.
+ The dilation factor for each dimension of input. If set to k > 1,
+ there will be k-1 skipped cells between each filter element on that dimension.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+ pass
+
+
+def conv2d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NHWC', dilations=None, name=None
+):
+ """
+ The transpose of conv2d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 4-D Tensor of type float and shape [batch, height, width, in_channels]
+ for NHWC data format or [batch, in_channels, height, width] for NCHW data format.
+ filters : tensor
+ A 4-D Tensor with the same type as input and shape [height, width,
+ output_channels, in_channels]. filter's in_channels dimension must match that of input.
+ output_shape : tensor
+ A 1-D Tensor representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1, 2 or 4. The stride of the sliding window for each dimension of input.
+ If a single value is given it is replicated in the H and W dimension.
+ By default the N and C dimensions are set to 0.
+ The dimension order is determined by the value of data_format, see below for details.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NHWC' and 'NCHW' are supported.
+ dilations : list
+ An int or list of ints that has length 1, 2 or 4, defaults to 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as input.
+ """
+ pass
+
+
+def conv3d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NDHWC', dilations=None, name=None
+):
+ """
+ The transpose of conv3d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of type float and shape [batch, height, width, in_channels] for
+ NHWC data format or [batch, in_channels, height, width] for NCHW data format.
+ filters : tensor
+ A 5-D Tensor with the same type as value and shape [height, width, output_channels, in_channels].
+ filter's in_channels dimension must match that of value.
+ output_shape : tensor
+ A 1-D Tensor representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1, 3 or 5.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NDHWC' and 'NCDHW' are supported.
+ dilations : list of ints
+ An int or list of ints that has length 1, 3 or 5, defaults to 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ pass
+
+
+class BatchNorm(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, *args, **kwargs):
+ pass
diff --git a/tensorlayer/backend/ops/load_backend.py b/tensorlayer/backend/ops/load_backend.py
new file mode 100644
index 0000000..4343507
--- /dev/null
+++ b/tensorlayer/backend/ops/load_backend.py
@@ -0,0 +1,81 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import json
+import os
+import sys
+
+BACKEND = 'tensorflow'
+# BACKEND = 'mindspore'
+# BACKEND = 'dragon'
+
+# Check for backend.json files
+tl_backend_dir = os.path.expanduser('~')
+if not os.access(tl_backend_dir, os.W_OK):
+ tl_backend_dir = '/tmp'
+tl_dir = os.path.join(tl_backend_dir, '.tl')
+
+config = {
+ 'backend': BACKEND,
+}
+if not os.path.exists(tl_dir):
+ path = os.path.join(tl_dir, 'tl_backend.json')
+ os.makedirs(tl_dir)
+ with open(path, "w") as f:
+ json.dump(config, f)
+ BACKEND = config['backend']
+ sys.stderr.write("Create the backend configuration file :" + path + '\n')
+else:
+ path = os.path.join(tl_dir, 'tl_backend.json')
+ with open(path, 'r') as load_f:
+ load_dict = json.load(load_f)
+ if load_dict['backend'] is not config['backend']:
+ BACKEND = config['backend']
+ else:
+ BACKEND = load_dict['backend']
+
+# Set backend based on TL_BACKEND flag.
+if 'TL_BACKEND' in os.environ:
+ backend = os.environ['TL_BACKEND']
+ if backend:
+ BACKEND = backend
+
+# import backend functions
+if BACKEND == 'tensorflow':
+ from .tensorflow_backend import *
+ from .tensorflow_nn import *
+ import tensorflow as tf
+ BACKEND_VERSION = tf.__version__
+ sys.stderr.write('Using TensorFlow backend.\n')
+
+elif BACKEND == 'mindspore':
+ from .mindspore_backend import *
+ from .mindspore_nn import *
+ import mindspore as ms
+ BACKEND_VERSION = ms.__version__
+ # set context
+ import mindspore.context as context
+ import os
+ os.environ['DEVICE_ID'] = '0'
+ #context.set_context(mode=context.PYNATIVE_MODE,device_target='GPU'),
+ context.set_context(mode=context.GRAPH_MODE, device_target='CPU'),
+ # enable_task_sink=True, enable_loop_sink=True)
+ # context.set_context(mode=context.GRAPH_MODE, backend_policy='ms',
+ # device_target='Ascend', enable_task_sink=True, enable_loop_sink=True)
+ sys.stderr.write('Using MindSpore backend.\n')
+
+elif BACKEND == 'dragon':
+ from .dragon_backend import *
+ from .dragon_nn import *
+ import dragon as dg
+ BACKEND_VERSION = dg.__version__
+ sys.stderr.write('Using Dragon backend.\n')
+
+elif BACKEND == 'paddle':
+ from .paddle_backend import *
+ from .paddle_nn import *
+ import paddle as pd
+ BACKEND_VERSION = pd.__version__
+ sys.stderr.write('Using Paddle backend.\n')
+else:
+ raise NotImplementedError("This backend is not supported")
diff --git a/tensorlayer/backend/ops/mindspore_backend.py b/tensorlayer/backend/ops/mindspore_backend.py
new file mode 100644
index 0000000..d067be2
--- /dev/null
+++ b/tensorlayer/backend/ops/mindspore_backend.py
@@ -0,0 +1,1239 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, division, print_function
+from .mindspore_nn import nchw_to_nhwc, nhwc_to_nchw
+from mindspore._c_expression.typing import Type
+from mindspore.common import dtype as mstype
+
+from mindspore.common.parameter import Parameter
+from mindspore.common.initializer import (
+ initializer, Constant, Normal, TruncatedNormal, Initializer, _assignment, _calculate_in_and_out, One, Zero
+)
+from mindspore.common.tensor import Tensor
+from mindspore._c_expression import Tensor as Tensor_
+from mindspore.ops import operations as P
+from mindspore.ops import functional as F
+from mindspore.ops import composite as C
+import mindspore.context as context
+from mindspore.nn import Cell
+
+import numpy as np
+from scipy.stats import truncnorm
+import random
+
+_dtypeDict = {
+ 'DType': Type,
+ 'float16': mstype.float16,
+ 'float32': mstype.float32,
+ 'float64': mstype.float64,
+ 'int8': mstype.int8,
+ 'int16': mstype.int16,
+ 'int32': mstype.int32,
+ 'int64': mstype.int64,
+ 'uint8': mstype.uint8,
+ 'uint16': mstype.uint16,
+ 'uint32': mstype.uint32,
+ 'uint64': mstype.uint64
+}
+
+DType = Type
+float16 = mstype.float16
+float32 = mstype.float32
+float64 = mstype.float64
+int8 = mstype.int8
+int16 = mstype.int16
+int32 = mstype.int32
+int64 = mstype.int64
+uint8 = mstype.uint8
+uint16 = mstype.uint16
+uint32 = mstype.uint32
+uint64 = mstype.uint64
+
+# isinstance input output
+# TensorLike = Tensor_
+
+
+def set_context(**kwargs):
+ return context.set_context(**kwargs)
+
+
+def get_tensor_shape(x):
+ return list(P.Shape()(x))
+
+
+# initializers
+def zeros(shape, dtype=mstype.float32):
+ """
+ Creates a tensor with all elements set to zero.
+
+ Parameters
+ ----------
+ shape : A list of integers
+ a tuple of integers, or a 1-D Tensor of type int32.
+ dtype : tensor
+ The DType of an element in the resulting Tensor
+
+ Returns
+ -------
+ A Tensor with all elements set to zero.
+
+ """
+ # shape = shape[::-1]
+ arr = np.ndarray(shape)
+ init_obj = Zero()
+ init_obj(arr)
+ return Tensor(arr, dtype=dtype)
+
+
+def ones(shape, dtype=mstype.float32):
+ """
+ Creates a tensor with all elements set to ones.
+
+ Parameters
+ ----------
+ shape : A list of integers
+ a tuple of integers, or a 1-D Tensor of type int32.
+ dtype : tensor
+ The DType of an element in the resulting Tensor
+
+ Returns
+ -------
+ A Tensor with all elements set to zero.
+
+ """
+ # shape = shape[::-1]
+ arr = np.ndarray(shape)
+ init_obj = One()
+ init_obj(arr)
+ return Tensor(arr, dtype=dtype)
+
+
+def constant(value, dtype=mstype.float32, shape=None):
+ """
+ Creates a constant tensor from a tensor-like object.
+
+ Parameters
+ ----------
+ value : list
+ A constant value (or list) of output type dtype.
+ dtype : tensor
+ The type of the elements of the resulting tensor.
+ shape : tuple
+ Optional dimensions of resulting tensor.
+
+ Returns
+ -------
+ A Constant Tensor.
+
+ """
+ # shape = shape[::-1]
+ arr = np.ndarray(shape)
+ Constant(value)(arr=arr)
+ return Tensor(arr, dtype=dtype)
+
+
+class Uniform(Initializer):
+ """
+ Initialize a uniform array, and obtain values U(-scale, scale) from the uniform distribution
+ to fill the input tensor.
+
+ Args:
+ minval : int
+ The lower bound on the range of random values to generate (inclusive). Defaults to 0.
+ maxval : int
+ The upper bound on the range of random values to generate (exclusive). Defaults to 1 if dtype is floating point.
+ seed : int
+ Used in combination with tf.random.set_seed to create a reproducible sequence of tensors across multiple calls.
+
+ Returns:
+ Array, uniform array.
+ """
+
+ def __init__(self, minval=0, maxval=None, seed=None):
+ super(Uniform, self).__init__(minval=minval, maxval=maxval, seed=seed)
+ self.minval = minval
+ self.maxval = maxval
+ self.seed = seed
+
+ def _initialize(self, arr):
+ random.seed(self.seed)
+ tmp = np.random.uniform(self.minval, self.maxval, arr.shape)
+ _assignment(arr, tmp)
+
+
+def random_uniform(shape, minval=0, maxval=None, dtype=mstype.float32, seed=None):
+ """
+ Outputs random values from a uniform distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ minval : int
+ The lower bound on the range of random values to generate (inclusive). Defaults to 0.
+ maxval : int
+ The upper bound on the range of random values to generate (exclusive). Defaults to 1 if dtype is floating point.
+ dtype : tensor
+ The type of the output: float16, float32, float64, int32, or int64.
+ seed : int
+ Used in combination with tf.random.set_seed to create a reproducible sequence of tensors across multiple calls.
+ Returns
+ -------
+ A tensor of the specified shape filled with random uniform values.
+
+ """
+ # shape = shape[::-1]
+ arr = np.ndarray(shape)
+ init_obj = Uniform(minval=minval, maxval=maxval, seed=seed)
+ init_obj(arr)
+ return Tensor(arr, dtype=dtype)
+
+
+class Normal(Initializer):
+ """
+ Initialize a normal array, and obtain values N(0, sigma) from the uniform distribution
+ to fill the input tensor.
+
+ Parameters
+ ----------
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns:
+ Array, normal array.
+ """
+
+ def __init__(self, mean=0.0, stddev=0.01, seed=None):
+ super(Normal, self).__init__(mean=mean, stddev=stddev)
+ self.mean = mean
+ self.stddev = stddev
+ self.seed = seed
+
+ def _initialize(self, arr):
+ random.seed(self.seed)
+ tmp = np.random.normal(self.mean, self.stddev, arr.shape)
+ _assignment(arr, tmp)
+
+
+class RandomNormal(Cell):
+ def __init__(self, mean=0.0, stddev=0.01, seed=None):
+ super(RandomNormal, self).__init__()
+ self.normal = Normal(mean=mean, stddev=stddev, seed=seed)
+
+ def construct(self, shape):
+ arr = np.ndarray(shape)
+ outputs = self.normal(arr)
+ return outputs
+
+
+def random_normal(shape, mean=0.0, stddev=1.0, dtype=mstype.float32, seed=None):
+ """
+ Outputs random values from a normal distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ dtype : tensor
+ The type of the output.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns
+ -------
+ A tensor of the specified shape filled with random normal values.
+
+ """
+ # shape = shape[::-1]
+ arr = np.ndarray(shape)
+ init_obj = Normal(mean=mean, stddev=stddev, seed=seed)
+ init_obj(arr)
+ return Tensor(arr, dtype=dtype)
+
+
+class TruncatedNormal(Initializer):
+ """
+ Initialize a truncated normal distribution which is a bounded normal distribution within N(low, high).
+
+ Args:
+ sigma (float): The sigma of the array. Default: 0.01.
+
+ Returns:
+ Array, truncated normal array.
+ """
+
+ def __init__(self, mean=0.0, stddev=0.01, seed=None):
+ super(TruncatedNormal, self).__init__(mean=mean, stddev=stddev, seed=seed)
+ self.mean = mean
+ self.stddev = stddev
+ self.seed = seed
+
+ def _initialize(self, arr):
+ tmp = truncnorm.rvs(-2, 2, loc=self.mean, scale=self.stddev, size=arr.shape, random_state=None)
+ _assignment(arr, tmp)
+
+
+def truncated_normal(shape, mean=0.0, stddev=1.0, dtype=mstype.float32, seed=None):
+ """
+ Outputs random values from a truncated normal distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ dtype : tensor
+ The type of the output.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns
+ -------
+ A tensor of the specified shape filled with random truncated normal values.
+
+ """
+ # shape = shape[::-1]
+ arr = np.ndarray(shape)
+ init_obj = TruncatedNormal(mean=mean, stddev=stddev, seed=seed)
+ init_obj(arr)
+ return Tensor(arr, dtype=dtype)
+
+
+class HeNormal(Initializer):
+ r"""
+ he_normal: It draws samples from a truncated normal distribution centered on 0 with
+ stddev = sqrt(2 / fan_in) where fan_in is the number of input units in the weight tensor.
+
+ Args:
+ arr (Array): The array to be assigned.
+
+ Returns:
+ Array, assigned array.
+ """
+
+ def __init__(self, seed=None):
+ super(HeNormal, self).__init__(seed=seed)
+ self.seed = seed
+
+ def _initialize(self, arr):
+ n_in, _ = _calculate_in_and_out(arr)
+ boundary = np.sqrt(2.0 / n_in)
+ random.seed(self.seed)
+ data = np.random.normal(-boundary, boundary, arr.shape)
+ _assignment(arr, data)
+
+
+def he_normal(shape, dtype, seed=None):
+ """
+ He normal initializer.
+
+ Parameters
+ ----------
+ seed : A Python integer.
+ Used to seed the random generator.
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ dtype : tensor
+ The type of the output.
+
+ Returns
+ -------
+ A tensor of the specified shape filled with he normal values.
+ """
+ # shape = shape[::-1]
+ arr = np.ndarray(shape)
+ init_obj = HeNormal(seed)
+ init_obj(arr)
+ return Tensor(arr, dtype=dtype)
+
+
+def Variable(initial_value, name, trainable=True):
+ """
+ Creates a new variable with value initial_value.
+
+ Parameters
+ ----------
+ initial_value : tensor
+ A Tensor, or Python object convertible to a Tensor
+ name : str
+ Optional name for the variable. Defaults to 'Variable' and gets uniquified automatically.
+ Returns
+ -------
+ Variable
+ """
+
+ var = Parameter(initial_value, name=name, requires_grad=trainable)
+ return var
+
+
+class MatMul(Cell):
+
+ def __init__(self):
+ super(MatMul, self).__init__()
+ self.matmul = P.MatMul()
+
+ def construct(self, a, b):
+ return self.matmul(a, b)
+
+
+def matmul(a, b):
+ """
+ Multiplies matrix a by matrix b, producing a * b.
+
+ Parameters
+ ----------
+ a : tensor
+ type float16, float32, float64, int32, complex64, complex128 and rank > 1.
+ b : tensor
+ with same type and rank as a.
+
+ Returns
+ -------
+ A Tensor of the same type as a and b
+ """
+ matmul_obj = P.MatMul()
+ outputs = matmul_obj(a, b)
+ return outputs
+
+
+def add(value, bias):
+ """
+ Returns x + y element-wise.
+
+ Parameters
+ ----------
+ value : tensor.
+ Must be one of the following types: bfloat16, half, float32, float64,
+ uint8, int8, int16, int32, int64, complex64, complex128, string.
+ bias : tensor
+ Must have the same type as a
+ name : str
+ A name for the operation
+
+ Returns
+ -------
+ A Tensor. Has the same type as a.
+ """
+
+ add_obj = P.TensorAdd()
+ outputs = add_obj(value, bias)
+ return outputs
+
+
+def dtypes(dt):
+ """
+ Data dtypes.
+
+ Parameters
+ ----------
+ dt : string
+ It could be 'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16',
+ 'int32', 'int64', 'float16', 'float32', 'float64', 'DType'.
+
+ Returns
+ -------
+ Data dtypes
+ """
+
+ if dt not in _dtypeDict.keys():
+ raise Exception("Unsupported dtype: {}".format(dt))
+ return _dtypeDict[dt]
+
+
+class Maximum(Cell):
+
+ def __init__(self):
+ super(Maximum, self).__init__()
+ self.maximum = P.Maximum()
+
+ def construct(self, x, y):
+ return self.maximum(x, y)
+
+
+class Minimum(Cell):
+
+ def __init__(self):
+ super(Minimum, self).__init__()
+ self.minimum = P.Minimum()
+
+ def construct(self, x, y):
+ return self.minimum(x, y)
+
+
+def minimum(x, y):
+ """
+ Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+
+ Parameters
+ ----------
+ x : tensor.
+ Must be one of the following types: bfloat16, half, float32, float64, int32, int64.
+ y : A Tensor.
+ Must have the same type as x.
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor. Has the same type as x
+ """
+ minimum_obj = P.Minimum()
+ outputs = minimum_obj(x, y)
+ return outputs
+
+
+class FlattenReshape(Cell):
+
+ def __init__(self):
+ super(FlattenReshape, self).__init__()
+ self.shape = P.Shape()
+ self.reshape = P.Reshape()
+
+ def construct(self, inputs):
+ dim = 1
+ for d in self.shape(inputs)[1:]:
+ dim *= d
+ return self.reshape(inputs, (-1, dim))
+
+
+class Reshape(Cell):
+
+ def __init__(self, shape):
+ super(Reshape, self).__init__()
+ self.reshape = P.Reshape()
+ self.shape = tuple(shape)
+
+ def construct(self, tensor):
+ return self.reshape(tensor, self.shape)
+
+
+def reshape(tensor, shape):
+ """
+ Reshapes a tensor.
+
+ Parameters
+ ----------
+ tensor : tensor
+ A Tensor.
+ shape : tensor
+ Defines the shape of the output tensor.
+ Returns
+ -------
+ A Tensor. Has the same type as tensor
+ """
+ reshape_obj = P.Reshape()
+ outputs = reshape_obj(tensor, tuple(shape))
+ return outputs
+
+
+class Concat(Cell):
+
+ def __init__(self, axis):
+ super(Concat, self).__init__()
+ self.concat = P.Concat(axis)
+
+ def construct(self, values):
+ return self.concat(values)
+
+
+def concat(values, axis):
+ """
+ Concatenates tensors along one dimension.
+
+ Parameters
+ ----------
+ values : list
+ A list of Tensor objects or a single Tensor
+ axis : int
+ 0-D int32 Tensor. Dimension along which to concatenate
+ Returns
+ -------
+ A Tensor resulting from concatenation of the input tensors.
+ """
+ # TODO testing axis
+ concat_obj = P.Concat(axis)
+ outputs = concat_obj(values)
+ return outputs
+
+
+def convert_to_tensor(value, dtype=None):
+ """
+ Converts the given value to a Tensor.
+
+ Parameters
+ ----------
+ value : object
+ An object whose type has a registered Tensor conversion function.
+ dtype : optional
+ Optional element type for the returned tensor. If missing, the type is inferred from the type of value.
+
+ Returns
+ -------
+ A Tensor based on value.
+ """
+ #todo testing value
+ return Tensor(value, dtype=dtype)
+
+
+def sqrt(x):
+ """
+ Computes square root of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x.
+ """
+ sqrt_obj = P.Sqrt()
+ outputs = sqrt_obj(x)
+ return outputs
+
+
+class ReduceSum(Cell):
+
+ def __init__(self, axis):
+ super(ReduceSum, self).__init__()
+ self.axis = axis
+ self.reduce_sum = P.ReduceSum(keep_dims=True)
+
+ def construct(self, input):
+ return self.reduce_sum(input, self.axis)
+
+
+class ReduceMean(Cell):
+
+ def __init__(self, axis):
+ super(ReduceMean, self).__init__()
+ self.axis = axis
+ self.reducemean = P.ReduceMean(keep_dims=False)
+
+ def construct(self, inputs):
+ output = self.reducemean(inputs, self.axis)
+ return output
+
+
+def reduce_mean(input_tensor, axis=None):
+ """
+ Computes the mean of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ Rmean_obj = P.ReduceMean(keep_dims=False)
+ outputs = Rmean_obj(input_tensor, axis)
+ return outputs
+
+
+class ReduceMax(Cell):
+
+ def __init__(self, axis):
+ super(ReduceMax, self).__init__()
+ self.axis = axis
+ self.reducemax = P.ReduceMax(keep_dims=False)
+
+ def construct(self, inputs):
+ output = self.reducemax(inputs, self.axis)
+ return output
+
+
+def reduce_max(input_tensor, axis=None):
+ """
+ Computes the maximum of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have real numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ Rmax_obj = P.ReduceMax(keep_dims=False)
+ outputs = Rmax_obj(input_tensor, axis)
+ return outputs
+
+
+def reduce_min(input_tensor, axis=None):
+ """
+ Computes the minimum of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have real numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ Rmin_obj = P.ReduceMin(keep_dims=False)
+ outputs = Rmin_obj(input_tensor, axis)
+ return outputs
+
+class Pad(Cell):
+ def __init__(self, paddings, mode="REFLECT"):
+ super(Pad, self).__init__()
+ if mode not in ["REFLECT", "SYMMETRIC"]:
+ raise Exception("Unsupported mode: {}".format(mode))
+ self.pad = P.MirrorPad(mode=mode)
+ self.paddings = Tensor(paddings)
+
+ def construct(self, x):
+ return self.pad(x, self.paddings)
+
+def pad(tensor, paddings, mode='CONSTANT', constant_values=0):
+ """
+ Pads a tensor.
+
+ Parameters
+ ----------
+ tensor : tensor
+ A Tensor.
+ paddings : tuple
+ A tuple of type int32.
+ mode : str
+ One of "CONSTANT", "REFLECT", or "SYMMETRIC" (case-insensitive)
+ constant_values : int
+ In "CONSTANT" mode, the scalar pad value to use. Must be same type as tensor.
+
+ Returns
+ -------
+ A Tensor. Has the same type as tensor.
+ """
+ raise NotImplementedError
+
+
+class Unstack(Cell):
+ def __init__(self, axis, num=None):
+ super(Unstack, self).__init__()
+ if num is not None:
+ raise ("The num Parameters do not need to be set.")
+ self.unstack = P.Unpack(axis=axis)
+
+ def construct(self, values):
+ return self.unstack(values)
+
+
+class Stack(Cell):
+ def __init__(self, axis=0):
+ super(Stack, self).__init__()
+ self.stack = P.Pack(axis=axis)
+
+ def construct(self, values):
+ return self.stack(values)
+
+
+def stack(values, axis=0):
+ """
+ Stacks a list of rank-R tensors into one rank-(R+1) tensor.
+
+ Parameters
+ ----------
+ values : list
+ A list of Tensor objects with the same shape and type.
+ axis : int
+ An int. The axis to stack along. Defaults to the first dimension.
+ Negative values wrap around, so the valid range is [-(R+1), R+1).
+
+ Returns
+ -------
+ A stacked Tensor with the same type as values.
+ """
+ _stack = P.Pack(axis=axis)
+ return _stack(values)
+
+
+class Meshgrid(Cell):
+ def __init__(self, indexing='xy'):
+ super(Meshgrid, self).__init__()
+ self._meshgrid = P.Meshgrid(indexing=indexing)
+
+ def construct(self, *args):
+ inputs = tuple(*args)
+ return self._meshgrid(inputs)
+
+
+
+def meshgrid(*args, **kwargs):
+ """
+ Broadcasts parameters for evaluation on an N-D grid.
+
+ Parameters
+ ----------
+ x : tensor
+ Tensors with rank 1.
+ y : tensor
+ Tensors with rank 1.
+
+ Returns
+ -------
+ A list of N Tensors with rank N.
+ """
+
+ _meshgrid = P.Meshgrid(**kwargs)
+ return _meshgrid(*args)
+
+
+
+def range(start, limit=None, delta=1, dtype=None):
+ """
+ Creates a sequence of numbers.
+
+ Parameters
+ ----------
+ start : tensor
+ A 0-D Tensor (scalar). Acts as first entry in the range if limit is not None;
+ otherwise, acts as range limit and first entry defaults to 0.
+ limit : tensor
+ A 0-D Tensor (scalar). Upper limit of sequence, exclusive. If None,
+ defaults to the value of start while the first entry of the range defaults to 0.
+ delta : tensor
+ A 0-D Tensor (scalar). Number that increments start. Defaults to 1.
+ dtype : type
+ The type of the elements of the resulting tensor.
+
+ Returns
+ -------
+ An 1-D Tensor of type dtype.
+ """
+
+ pass
+
+
+class ExpandDims(Cell):
+
+ def __init__(self, axis):
+ super(ExpandDims, self).__init__()
+ self.axis = axis
+ self.expand_dims = P.ExpandDims()
+
+ def construct(self, input):
+ output = self.expand_dims(input, self.axis)
+ return output
+
+
+def expand_dims(input, axis):
+ """
+ Inserts a dimension of 1 into a tensor's shape.
+
+ Parameters
+ ----------
+ input : tensor
+ A Tensor.
+ axis : int
+ 0-D (scalar). Specifies the dimension index at which to expand the shape of input.
+ Must be in the range [-rank(input) - 1, rank(input)].
+
+ Returns
+ -------
+ A Tensor with the same data as input, but its shape has an additional dimension of size 1 added.
+ """
+
+ expand_obj = P.ExpandDims()
+ outputs = expand_obj(input, axis)
+ return outputs
+
+
+class Tile(Cell):
+
+ def __init__(self):
+ super(Tile, self).__init__()
+ self.tile = P.Tile()
+
+ def construct(self, input, multiples):
+ return self.tile(input, tuple(multiples))
+
+
+
+def tile(input, multiples):
+ """
+ Constructs a tensor by tiling a given tensor.
+
+ Parameters
+ ----------
+ input : tensor
+ A Tensor. 1-D or higher.
+ multiples : tensor
+ Must be one of the following types: int32, int64. 1-D.
+ Length must be the same as the number of dimensions in input
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ tile_obj = P.Tile()
+ outputs = tile_obj(input, multiples)
+ return outputs
+
+
+class Cast(Cell):
+
+ def __init__(self, dtype):
+ super(Cast, self).__init__()
+ self.dtype = dtype
+ self.cast = P.Cast()
+
+ def construct(self, input):
+ return self.cast(input, dtype=self.dtype)
+
+
+def cast(x, dtype):
+ """
+ Casts a tensor to a new type.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor or SparseTensor or IndexedSlices of numeric type.
+ It could be uint8, uint16, uint32, uint64, int8, int16, int32, int64, float16, float32, float64.
+ dtype : dtpye
+ The destination type. The list of supported dtypes is the same as x
+
+ Returns
+ -------
+ A Tensor or SparseTensor or IndexedSlices with same shape as x and same type as dtype.
+ """
+ cast_obj = P.Cast()
+ outputs = cast_obj(x, dtype)
+ return outputs
+
+
+class Transpose(Cell):
+
+ def __init__(self, perm, conjugate=False):
+ super(Transpose, self).__init__()
+ self.perm = tuple(perm)
+ self.conjugate = conjugate
+ self.transpose = P.Transpose()
+ if self.conjugate:
+ raise NotImplementedError("conjugate not implemented")
+
+ def construct(self, a):
+ return self.transpose(a, self.perm)
+
+
+def transpose(a, perm=None, conjugate=False):
+ """
+ Transposes a.
+
+ Parameters
+ ----------
+ a : tensor
+ A Tensor.
+ perm : int
+ A permutation of the dimensions of a.
+ conjugate : bool
+ Setting it to True is mathematically equivalent to ms.math.conj(ms.transpose(input)).
+
+ Returns
+ -------
+ A transposed Tensor.
+ """
+ # TODO conjugate
+ trans_obj = P.Transpose()
+ outputs = trans_obj(a, perm)
+ print(outputs)
+
+
+def gather_nd(params, indices, batch_dims=0):
+ """
+ Gather slices from params into a Tensor with shape specified by indices.
+
+ Parameters
+ ----------
+ params : tensor
+ The tensor from which to gather values.
+ indices : tensor
+ Must be one of the following types: int32, int64. Index tensor.
+ batch_dims : int
+ An integer or a scalar 'Tensor'. The number of batch dimensions.
+
+ Returns
+ -------
+ A Tensor. Has the same type as params.
+ """
+
+ pass
+
+
+def clip_by_value(t, clip_value_min, clip_value_max):
+ """
+ Clips tensor values to a specified min and max.
+
+ Parameters
+ ----------
+ t : tensor
+ A Tensor or IndexedSlices
+ clip_value_min : tensor
+ A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by
+ clip_value_max : tensor
+ A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by
+
+ Returns
+ -------
+ A clipped Tensor or IndexedSlices.
+ """
+ min_value = Tensor(clip_value_min, mstype.float32)
+ max_value = Tensor(clip_value_max, mstype.float32)
+ output = C.clip_by_value(t, min_value, max_value)
+ return output
+
+
+def split(value, num_or_size_splits, axis=0, num=None):
+ """
+ Splits a tensor into sub tensors.
+
+ Parameters
+ ----------
+ value : tensor
+ The Tensor to split.
+ num_or_size_splits : list
+ Either an integer indicating the number of splits along split_dim or a 1-D integer Tensor or
+ Python list containing the sizes of each output tensor along split_dim.
+ axis : int
+ The dimension along which to split. Must be in the range [-rank(value), rank(value)). Defaults to 0.
+ num : int
+ used to specify the number of outputs when it cannot be inferred from the shape of size_splits.
+
+ Returns
+ -------
+ Tensor objects resulting from splitting value.
+ """
+ pass
+
+
+def floor(x):
+ return NotImplementedError
+
+
+def gather(params, indices):
+ return NotImplementedError
+
+
+def linspace(start, stop, num):
+ return NotImplementedError
+
+
+def slice(inputs, starts, sizes):
+ return NotImplementedError
+
+
+def add_n(inputs):
+ return NotImplementedError
+
+
+class OneHot(Cell):
+
+ def __init__(self, axis=-1, depth=1, on_value=1.0, off_value=0.0, dtype=mstype.float32):
+ super(OneHot, self).__init__()
+ self.onehot = P.OneHot(axis)
+ self.depth = depth
+ self.dtype = dtype
+ self.on_value = F.cast(on_value, self.dtype)
+ self.off_value = F.cast(off_value, self.dtype)
+
+ def construct(self, indices):
+ return self.onehot(indices, self.depth, self.on_value, self.off_value)
+
+
+class L2Normalize(Cell):
+
+ def __init__(self, axis=None, epsilon=1e-12):
+ super(L2Normalize, self).__init__()
+ pass
+
+ def __call__(self, input, *args, **kwargs):
+ pass
+
+
+class EmbeddingLookup(Cell):
+
+ def __init__(self, max_norm=None):
+ self.max_norm = max_norm
+
+ def __call__(self, params, ids, *args, **kwargs):
+ pass
+
+
+class NCELoss(object):
+
+ def __init__(self, num_true=1, sampled_values=None, remove_accidental_hits=False):
+ super(NCELoss, self).__init__()
+
+ def __call__(self, weights, biases, labels, inputs, num_sampled, num_classes):
+ pass
+
+
+class Not_equal(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x, y):
+ pass
+
+
+class Count_nonzero(object):
+
+ def __init__(self, keepdims=None, dtype=int64):
+ pass
+
+ def __call__(self, *args, **kwargs):
+ pass
+
+
+class Resize(Cell):
+
+ def __init__(self, scale, method, antialias=False, data_format='channels_last', ksize=None):
+ super(Resize, self).__init__()
+ self.data_format = data_format
+ if method not in ['nearest', 'bilinear']:
+ raise ('The method must be "nearest" or "bilinear".')
+ self.method = method
+
+ if ksize is None:
+ raise ('The "bilinear" and "nearest" method must enter ksize. The dimension of size must be 2 (H, W).')
+
+ out_seize = (int(ksize[0] * scale[0]), int(ksize[1] * scale[1]))
+ if self.method == 'nearest':
+ self.resize = P.ResizeNearestNeighbor(size=out_seize, align_corners=antialias)
+ elif self.method == 'bilinear':
+
+ self.resize = P.ResizeBilinear(size=out_seize)
+
+ def construct(self, inputs):
+ if self.data_format == 'channels_last':
+ inputs = nhwc_to_nchw(inputs)
+ outputs = self.resize(inputs)
+ if self.data_format == 'channels_last':
+ outputs = nchw_to_nhwc(outputs)
+ return outputs
+
+
+def resize(inputs, output_size, method, antialias):
+ raise NotImplementedError
+
+
+class ZeroPadding1D(Cell):
+ def __init__(self, padding):
+ super(ZeroPadding1D, self).__init__()
+ if np.size(padding) == 2:
+ self.pad = P.Pad(paddings=padding)
+ else:
+ raise ("The shape of parameter paddings is (N, 2). N is the rank of input data.")
+
+ def construct(self, inputs):
+ return self.pad(inputs)
+
+
+class ZeroPadding2D(Cell):
+ def __init__(self, padding):
+ super(ZeroPadding2D, self).__init__()
+ if np.size(padding) == 4:
+ self.pad = P.Pad(paddings=padding)
+ else:
+ raise ("The shape of parameter paddings is (N, 2). N is the rank of input data.")
+
+ def construct(self, inputs):
+ return self.pad(inputs)
+
+
+class ZeroPadding3D(Cell):
+ def __init__(self, padding):
+ super(ZeroPadding3D, self).__init__()
+ if np.size(padding) == 6:
+ self.pad = P.Pad(paddings=padding)
+ else:
+ raise ("The shape of parameter paddings is (N, 2). N is the rank of input data.")
+
+ def construct(self, inputs):
+ return self.pad(inputs)
+
+
+class Sign(Cell):
+
+ def __init__(self):
+ super(Sign, self).__init__()
+ self.sign = P.Sign()
+
+ def construct(self, x):
+ return self.sign(x)
+
+def ceil(x):
+ _ceil = P.Ceil()
+ return _ceil(x)
+
+def multiply(x, y):
+ raise NotImplementedError
+
+def divide(x, y):
+ raise NotImplementedError
+
+def identity(x):
+ raise NotImplementedError
+
+class BatchToSpace(Cell):
+ def __init__(self, block_size, crops):
+ super(BatchToSpace, self).__init__()
+ self.batch_to_space = P.BatchToSpace(block_size=block_size, crops=crops)
+
+ def __call__(self, input_x):
+ return self.batch_to_space(input_x)
+
+class DepthToSpace(Cell):
+ def __init__(self, block_size, data_format='NHWC'):
+ super(DepthToSpace, self).__init__()
+ self.data_format = data_format
+ self.depth_to_space = P.DepthToSpace(block_size=block_size)
+
+ def __call__(self, input):
+ if self.data_format == 'NHWC':
+ input = nhwc_to_nchw(input)
+
+ output = self.depth_to_space(input)
+
+ if self.data_format == 'NHWC':
+ output = nchw_to_nhwc(output)
+
+ return output
\ No newline at end of file
diff --git a/tensorlayer/backend/ops/mindspore_nn.py b/tensorlayer/backend/ops/mindspore_nn.py
new file mode 100644
index 0000000..3af4430
--- /dev/null
+++ b/tensorlayer/backend/ops/mindspore_nn.py
@@ -0,0 +1,1323 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, division, print_function
+
+from mindspore.nn.cell import Cell
+from mindspore import context
+import mindspore as ms
+from mindspore.ops import operations as P
+from mindspore.ops import functional as F
+from mindspore.communication.management import get_group_size, get_rank
+from mindspore.communication import management
+from mindspore._checkparam import check_int_positive
+from mindspore._extends import cell_attr_register
+
+
+
+def padding_format(padding):
+ """
+ Checks that the padding format correspond format.
+
+ Parameters
+ ----------
+ padding : str
+ Must be one of the following:"same", "SAME", "VALID", "valid"
+
+ Returns
+ -------
+ str "SAME" or "VALID"
+ """
+
+ if padding in ["SAME", "same"]:
+ padding = "same"
+ elif padding in ["VALID", "valid"]:
+ padding = "valid"
+ elif padding == None:
+ padding = None
+ else:
+ raise Exception("Unsupported padding: " + str(padding))
+ return padding
+
+
+def preprocess_1d_format(data_format, padding):
+ """
+ Checks that the 1-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NWC","NCW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NWC" or "NCW" and "SAME" or "VALID"
+ """
+
+ if data_format in ["channels_last", "NWC"]:
+ data_format = "NWC"
+ elif data_format in ["channels_first", "NCW"]:
+ data_format = "NCW"
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def preprocess_2d_format(data_format, padding):
+ """
+ Checks that the 2-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NHWC","NCHW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NHWC" or "NCHW" and "SAME" or "VALID"
+ """
+
+ if data_format in ["channels_last", "NHWC", "nhwc"]:
+ data_format = "NHWC"
+ elif data_format in ["channels_first", "NCHW", "nchw"]:
+ data_format = "NCHW"
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def preprocess_3d_format(data_format, padding):
+ """
+ Checks that the 3-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NDHWC","NCDHW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NDHWC" or "NCDHW" and "SAME" or "VALID"
+ """
+
+ if data_format in ['channels_last', 'NDHWC']:
+ data_format = 'NDHWC'
+ elif data_format in ['channels_first', 'NCDHW']:
+ data_format = 'NCDHW'
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def nchw_to_nhwc(x):
+ """
+ Channels first to channels last
+
+ Parameters
+ ----------
+ x : tensor
+ channels first tensor data
+
+ Returns
+ -------
+ channels last tensor data
+ """
+
+ if len(P.Shape()(x)) == 3:
+ x = P.Transpose()(x, (0, 2, 1))
+ elif len(P.Shape()(x)) == 4:
+ x = P.Transpose()(x, (0, 2, 3, 1))
+ elif len(P.Shape()(x)) == 5:
+ x = P.Transpose()(x, (0, 2, 3, 4, 1))
+ # else:
+ # raise Exception("Unsupported dimensions")
+ return x
+
+
+def nhwc_to_nchw(x):
+ """
+ Channles last to channels first
+
+ Parameters
+ ----------
+ x : tensor
+ channels last tensor data
+
+ Returns
+ -------
+ channels first tensor data
+ """
+
+ if len(P.Shape()(x)) == 3:
+ x = P.Transpose()(x, (0, 2, 1))
+ elif len(P.Shape()(x)) == 4:
+ x = P.Transpose()(x, (0, 3, 1, 2))
+ elif len(P.Shape()(x)) == 5:
+ x = P.Transpose()(x, (0, 4, 1, 2, 3))
+ # else:
+ # raise Exception("Unsupported dimensions")
+ return x
+
+
+class ReLU(Cell):
+
+ def __init__(self):
+ super(ReLU, self).__init__()
+ self.relu = P.ReLU()
+
+ def construct(self, x):
+ return self.relu(x)
+
+
+def relu(x):
+ """
+ Computes rectified linear: max(features, 0).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: float32, float64, int32, uint8, int16,
+ int8, int64, bfloat16, uint16, half, uint32, uint64, qint8.
+
+ Returns
+ -------
+ A Tensor. Has the same type as features.
+ """
+ outputs = P.ReLU()
+ return outputs(x)
+
+
+class ReLU6(Cell):
+
+ def __init__(self):
+ super(ReLU6, self).__init__()
+ self.relu6 = P.ReLU6()
+
+ def construct(self, x):
+ return self.relu6(x)
+
+
+def relu6(x):
+ """
+ Computes Rectified Linear 6: min(max(features, 0), 6).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: float32, float64, int32, uint8, int16,
+ int8, int64, bfloat16, uint16, half, uint32, uint64, qint8.
+
+ Returns
+ -------
+ A Tensor with the same type as features.
+ """
+ outputs = P.ReLU6()
+ return outputs(x)
+
+
+class LeakyReLU(Cell):
+
+ def __init__(self, alpha=0.2):
+ super(LeakyReLU, self).__init__()
+ self.leakyrelu = ms.nn.LeakyReLU(alpha=alpha)
+
+ def construct(self, x):
+ return self.leakyrelu(x)
+
+
+def leaky_relu(x, alpha=0.2):
+ """
+ Compute the Leaky ReLU activation function.
+
+ Parameters
+ ----------
+ x : tensor
+ representing preactivation values. Must be one of the following types:
+ float16, float32, float64, int32, int64.
+
+ Returns
+ -------
+ The activation value.
+ """
+
+ leaky_relu = LeakyReLU(alpha=alpha)
+ output = leaky_relu(x)
+ return leaky_relu
+
+
+class Softplus(Cell):
+
+ def __init__(self):
+ super(Softplus, self).__init__()
+ self.softplus = P.Softplus()
+
+ def construct(self, x):
+ return self.softplus(x)
+
+
+def softplus(x):
+ """
+ Computes softplus: log(exp(features) + 1).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: half, bfloat16, float32, float64.
+
+ Returns
+ -------
+ A Tensor. Has the same type as features.
+ """
+
+ obj = Softplus()
+ return obj(x)
+
+
+class Tanh(Cell):
+
+ def __init__(self):
+ super(Tanh, self).__init__()
+ self.tanh = P.Tanh()
+
+ def construct(self, x):
+ return self.tanh(x)
+
+
+def tanh(x):
+ """
+ Computes hyperbolic tangent of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x.
+ """
+
+ _tanh = Tanh()
+ return _tanh(x)
+
+
+class Sigmoid(Cell):
+
+ def __init__(self):
+ super(Sigmoid, self).__init__()
+ self.sigmoid = P.Sigmoid()
+
+ def construct(self, x):
+ return self.sigmoid(x)
+
+
+def sigmoid(x):
+ """
+ Computes sigmoid of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float16, float32, float64, complex64, or complex128.
+
+ Returns
+ -------
+ A Tensor with the same type as x.
+ """
+ outputs = P.Sigmoid()
+ return outputs(x)
+
+
+class Softmax(Cell):
+
+ def __init__(self):
+ super(Softmax, self).__init__()
+ self.softmax = P.Softmax()
+
+ def construct(self, x):
+ return self.softmax(x)
+
+
+def softmax(logits, axis=None):
+ """
+ Computes softmax activations.
+
+ Parameters
+ ----------
+ logits : tensor
+ Must be one of the following types: half, float32, float64.
+ axis : int
+ The dimension softmax would be performed on. The default is -1 which indicates the last dimension.
+
+ Returns
+ -------
+ A Tensor. Has the same type and shape as logits.
+ """
+ outputs = P.Softmax(axis)
+ return outputs(logits)
+
+
+class Dropout(Cell):
+
+ def __init__(self, keep, seed=0):
+ super(Dropout, self).__init__()
+ self.dropout = P.Dropout(keep_prob=keep)
+ self.is_gpu = context.get_context('device_target') in ["GPU"]
+ self.get_shape = P.Shape()
+ self.dropout_gen_mask = P.DropoutGenMask(Seed0=seed, Seed1=0)
+ self.dropout_do_mask = P.DropoutDoMask()
+ self.cast = P.Cast()
+ self.keep_prob = keep # ms.Tensor(keep, dtype=ms.float32)
+ # print(self.keep_prob, type(self.keep_prob))
+
+ def construct(self, inputs):
+ if self.is_gpu:
+ outputs, _ = self.dropout(inputs)
+ return outputs
+ if self.keep_prob == 1:
+ return inputs
+ shape = self.get_shape(inputs)
+ dtype = P.DType()(inputs)
+ if self._is_float_dtype(dtype):
+ keep_prob = self.cast(self.keep_prob, dtype=dtype)
+ else:
+ keep_prob = self.cast(self.keep_prob, ms.float16)
+ output = self.dropout_gen_mask(shape, keep_prob)
+ return self.dropout_do_mask(inputs, output, keep_prob)
+
+ def _is_float_dtype(dtype):
+ if dtype in [ms.float32, ms.float16]:
+ return True
+ return False
+
+
+class BiasAdd(Cell):
+ """
+ Adds bias to value.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float, double, int64, int32, uint8, int16, int8, complex64, or complex128.
+ bias : tensor
+ Must be the same type as value unless value is a quantized type,
+ in which case a different quantized type may be used.
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ def __init__(self, data_format='channels_first'):
+ super(BiasAdd, self).__init__()
+ self.bias_add = P.BiasAdd()
+ if data_format in ['channels_first', 'NCW', 'NCHW', 'NCDHW']:
+ self.data_format = 'channels_first'
+ elif data_format in ['channels_last', 'NWC', 'NHWC', 'NDHWC']:
+ self.data_format = 'channels_last'
+ else:
+ raise ("Unsupported data format: " + str(data_format))
+
+ def construct(self, x, bias):
+ if self.data_format == 'channels_last':
+ x = nhwc_to_nchw(x)
+ outputs = self.bias_add(x, bias)
+ if self.data_format == 'channels_last':
+ outputs = nchw_to_nhwc(outputs)
+ return outputs
+
+
+def bias_add(x, bias):
+ """
+ Adds bias to value.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float, double, int64, int32, uint8, int16, int8, complex64, or complex128.
+ bias : tensor
+ Must be the same type as value unless value is a quantized type,
+ in which case a different quantized type may be used.
+ data_format : A string.
+ 'N...C' and 'NC...' are supported.
+ name : str
+ A name for the operation (optional).
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+ raise NotImplementedError
+
+
+class Conv1D(Cell):
+
+ def __init__(self, stride, padding, data_format='NWC', dilations=None, out_channel=None, k_size=None):
+ super(Conv1D, self).__init__()
+ self.data_format, self.padding = preprocess_1d_format(data_format, padding)
+ self.stride = (1, stride)
+ self.dilations = (1, dilations)
+ self.k_size = (1, k_size)
+ self.out_channel = out_channel
+
+ self.conv2d = P.Conv2D(
+ out_channel=self.out_channel, kernel_size=self.k_size, pad_mode=self.padding, stride=self.stride,
+ dilation=self.dilations, mode=1, group=1
+ )
+
+ self.expand_dims = P.ExpandDims()
+ self.squeeze = P.Squeeze(2)
+
+ def construct(self, x, filters):
+ if self.data_format == 'NWC':
+ x = nhwc_to_nchw(x)
+
+ x = self.expand_dims(x, 2)
+ filters = self.expand_dims(filters, 2)
+
+ output = self.conv2d(x, filters)
+ output = self.squeeze(output)
+
+ if self.data_format == 'NWC':
+ output = nchw_to_nhwc(output)
+ return output
+
+
+def conv1d(input, filters, stride, padding, data_format='NWC', dilations=None, name=None):
+ """
+ Computes a 1-D convolution given 3-D input and filter tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ A 3D Tensor. Must be of type float16, float32, or float64
+ filters : tensor
+ A 3D Tensor. Must have the same type as input.
+ stride : int of list
+ An int or list of ints that has length 1 or 3. The number of entries by which the filter is moved right at each step.
+ padding : string
+ 'SAME' or 'VALID'
+ data_format : string
+ An optional string from "NWC", "NCW". Defaults to "NWC", the data is stored in the order of
+ [batch, in_width, in_channels]. The "NCW" format stores data as [batch, in_channels, in_width].
+ dilations : int or list
+ An int or list of ints that has length 1 or 3 which defaults to 1.
+ The dilation factor for each dimension of input. If set to k > 1,
+ there will be k-1 skipped cells between each filter element on that dimension.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ A name for the operation (optional).
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ pass
+
+
+class Conv2D(Cell):
+
+ def __init__(self, strides, padding, data_format='NHWC', dilations=None, out_channel=None, k_size=None):
+ super(Conv2D, self).__init__()
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+
+ if self.data_format is 'NHWC':
+ self.ms_stride = strides[1]
+ self.ms_dilation = dilations[1]
+ # self.transpose = P.Transpose()
+ elif self.data_format is 'NCHW':
+ self.ms_stride = strides[2]
+ self.ms_dilation = dilations[2]
+
+ # print(out_channel, k_size, self.padding, self.ms_stride, self.ms_dilation)
+ self.conv2d = P.Conv2D(
+ out_channel=out_channel, kernel_size=k_size, pad_mode=self.padding, stride=self.ms_stride,
+ dilation=self.ms_dilation, mode=1, group=1
+ )
+
+ def construct(self, inputs, filters):
+ if self.data_format == 'NHWC':
+ inputs = nhwc_to_nchw(inputs)
+
+ outputs = self.conv2d(inputs, filters)
+
+ if self.data_format == 'NHWC':
+ outputs = nchw_to_nhwc(outputs)
+ return outputs
+
+
+def conv2d(input, filters, strides, padding, data_format='NCHW', dilations=None):
+ """
+ Computes a 2-D convolution given 4-D input and filters tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ Must be one of the following types: half, bfloat16, float32, float64. A 4-D tensor.
+ The dimension order is interpreted according to the value of data_format, see below for details.
+ filters : tensor
+ Must have the same type as input. A 4-D tensor of shape [filter_height, filter_width, in_channels, out_channels]
+ strides : int of list
+ The stride of the sliding window for each dimension of input. If a single value is given it is replicated in the H and W dimension.
+ By default the N and C dimensions are set to 1. The dimension order is determined by the value of data_format, see below for details.
+ padding : string
+ "SAME" or "VALID"
+ data_format : string
+ "NHWC", "NCHW". Defaults to "NCHW".
+ dilations : list or ints
+ list of ints that has length 1, 2 or 4, defaults to 1. The dilation factor for each dimension ofinput.
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ raise NotImplementedError
+
+
+class Conv3D(Cell):
+ pass
+ # raise NotImplementedError
+
+
+def conv3d(input, filters, strides, padding, data_format='NDHWC', dilations=None, name=None):
+ """
+ Computes a 3-D convolution given 5-D input and filters tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ Must be one of the following types: half, bfloat16, float32, float64.
+ Shape [batch, in_depth, in_height, in_width, in_channels].
+ filters : tensor
+ Must have the same type as input. Shape [filter_depth, filter_height, filter_width, in_channels, out_channels].
+ in_channels must match between input and filters.
+ strides : list of ints
+ A list of ints that has length >= 5. 1-D tensor of length 5.
+ The stride of the sliding window for each dimension of input.
+ Must have strides[0] = strides[4] = 1.
+ padding : string
+ A string from: "SAME", "VALID". The type of padding algorithm to use.
+ data_format : string
+ An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". The data format of the input and output data.
+ With the default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels].
+ Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width].
+ dilations : list of ints
+ Defaults to [1, 1, 1, 1, 1]. 1-D tensor of length 5. The dilation factor for each dimension of input.
+ If set to k > 1, there will be k-1 skipped cells between each filter element on that dimension.
+ The dimension order is determined by the value of data_format, see above for details.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ raise NotImplementedError
+
+
+def lrn(inputs, depth_radius, bias, alpha, beta):
+ """
+ Local Response Normalization.
+
+ Parameters
+ ----------
+ inputs : tensor
+ Must be one of the following types: half, bfloat16, float32. 4-D.
+ depth_radius : int
+ Defaults to 5. 0-D. Half-width of the 1-D normalization window.
+ bias : float
+ Defaults to 1. An offset (usually positive to avoid dividing by 0).
+ alpha : float
+ Defaults to 1. A scale factor, usually positive.
+ beta : float
+ Defaults to 0.5. An exponent.
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ pass
+
+
+def moments(x, axes, shift=None, keepdims=False):
+ """
+ Calculates the mean and variance of x.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor
+ axes : ints
+ Axes along which to compute mean and variance.
+ shift : int
+ Not used in the current implementation.
+ keepdims : bool
+ produce moments with the same dimensionality as the input.
+
+ Returns
+ -------
+ Two Tensor objects: mean and variance.
+ """
+
+ pass
+
+
+class MaxPool(Cell):
+
+ def __init__(self, ksize, strides, padding, data_format=None):
+ super(MaxPool, self).__init__()
+ self.data_format, self.padding = preprocess_2d_format(data_format=data_format, padding=padding)
+ ms_ksize = ksize[1]
+ ms_strides = strides[1]
+ self.maxpool = P.MaxPool(ksize=ms_ksize, strides=ms_strides, padding=self.padding)
+
+ def construct(self, inputs):
+ if self.data_format == 'NHWC':
+ inputs = nhwc_to_nchw(inputs)
+
+ outputs = self.maxpool(inputs)
+
+ if self.data_format == 'NHWC':
+ outputs = nchw_to_nhwc(outputs)
+ return outputs
+
+
+def max_pool(input, ksize, strides, padding, data_format=None):
+ """
+ Performs the max pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels] if data_format does not start
+ with "NC" (default), or [batch_size, num_channels] + input_spatial_shape if data_format starts with "NC".
+ Pooling happens over the spatial dimensions only.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The max pooled output tensor.
+ """
+ data_format, padding = preprocess_2d_format(data_format=data_format, padding=padding)
+ if data_format == 'NHWC':
+ input = nhwc_to_nchw(input)
+
+ ms_ksize = ksize[1]
+ ms_strides = strides[2]
+ outputs = P.MaxPool(ksize=ms_ksize, strides=ms_strides, padding=padding)(input)
+ # channel first to channel last
+ if data_format == 'NHWC':
+ outputs = nchw_to_nhwc(outputs)
+ return outputs
+
+
+class AvgPool(Cell):
+
+ def __init__(self, ksize, strides, padding, data_format=None):
+ super(AvgPool, self).__init__()
+ self.data_format, self.padding = preprocess_2d_format(data_format=data_format, padding=padding)
+ ms_ksize = ksize[1]
+ ms_strides = strides[1]
+ self.avgpool = P.AvgPool(ksize=ms_ksize, strides=ms_strides, padding=padding)
+
+ def construct(self, inputs):
+ if self.data_format == 'NHWC':
+ inputs = nhwc_to_nchw(inputs)
+
+ outputs = self.avgpool(inputs)
+
+ if self.data_format == 'NHWC':
+ outputs = nchw_to_nhwc(outputs)
+ return outputs
+
+
+def avg_pool(input, ksize, strides, padding):
+ """
+ Performs the avg pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels]
+ if data_format does not start with "NC" (default), or [batch_size, num_channels] + input_spatial_shape
+ if data_format starts with "NC". Pooling happens over the spatial dimensions only.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The average pooled output tensor.
+ """
+ padding = padding_format(padding)
+ ms_ksize = ksize[0]
+ ms_strides = strides[1]
+ outputs = P.AvgPool(ksize=ms_ksize, strides=ms_strides, padding=padding)
+ return outputs(input)
+
+
+def max_pool3d(input, ksize, strides, padding, data_format=None, name=None):
+ """
+ Performs the max pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of the format specified by data_format.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ "NDHWC", "NCDHW". Defaults to "NDHWC". The data format of the input and output data.
+ With the default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels].
+ Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width].
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The max pooled output tensor.
+ """
+ pass
+
+
+def avg_pool3d(input, ksize, strides, padding, data_format=None, name=None):
+ """
+ Performs the average pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of shape [batch, height, width, channels] and type float32, float64, qint8, quint8, or qint32.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, 3 or 5. The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NDHWC' and 'NCDHW' are supported.
+ name : string
+ Optional name for the operation.
+
+ Returns
+ -------
+ A Tensor with the same type as value. The average pooled output tensor.
+ """
+ pass
+
+
+def pool(input, window_shape, pooling_type, strides=None, padding='VALID', data_format=None, dilations=None, name=None):
+ """
+ Performs an N-D pooling operation.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels]
+ if data_format does not start with "NC" (default), or [batch_size, num_channels] + input_spatial_shape
+ if data_format starts with "NC". Pooling happens over the spatial dimensions only.
+ window_shape : int
+ Sequence of N ints >= 1.
+ pooling_type : string
+ Specifies pooling operation, must be "AVG" or "MAX".
+ strides : ints
+ Sequence of N ints >= 1. Defaults to [1]*N. If any value of strides is > 1, then all values of dilation_rate must be 1.
+ padding : string
+ The padding algorithm, must be "SAME" or "VALID". Defaults to "SAME".
+ See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ Specifies whether the channel dimension of the input and output is the last dimension (default, or if data_format does not start with "NC"),
+ or the second dimension (if data_format starts with "NC").
+ For N=1, the valid values are "NWC" (default) and "NCW". For N=2, the valid values are "NHWC" (default) and "NCHW".
+ For N=3, the valid values are "NDHWC" (default) and "NCDHW".
+ dilations : list of ints
+ Dilation rate. List of N ints >= 1. Defaults to [1]*N. If any value of dilation_rate is > 1, then all values of strides must be 1.
+ name : string
+ Optional. Name of the op.
+
+ Returns
+ -------
+ Tensor of rank N+2, of shape [batch_size] + output_spatial_shape + [num_channels]
+ """
+ pass
+
+
+
+class DepthwiseConv2d(Cell):
+
+ def __init__(self, strides, padding, data_format=None, dilations=None, ksize=None, channel_multiplier=1):
+ super(DepthwiseConv2d, self).__init__()
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.ms_stride = strides[1]
+ self.ms_dilation = dilations[1]
+ self.depthwise_conv2d = P.DepthwiseConv2dNative(
+ channel_multiplier=channel_multiplier, kernel_size=ksize, stride=self.ms_stride, dilation=self.ms_dilation
+ )
+
+ def construct(self, input, filter):
+ if self.data_format == 'NHWC':
+ input = nhwc_to_nchw(input)
+ outputs = self.depthwise_conv2d(input, filter)
+ if self.data_format == 'NHWC':
+ outputs = nchw_to_nhwc(outputs)
+ return outputs
+
+
+def depthwise_conv2d(input, filter, strides, padding, data_format=None, dilations=None, name=None):
+ """
+ Depthwise 2-D convolution.
+
+ Parameters
+ ----------
+ input : tensor
+ 4-D with shape according to data_format.
+ filter : tensor
+ 4-D with shape [filter_height, filter_width, in_channels, channel_multiplier].
+ strides : list
+ 1-D of size 4. The stride of the sliding window for each dimension of input.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ The data format for input. Either "NHWC" (default) or "NCHW".
+ dilations : list
+ 1-D of size 2. The dilation rate in which we sample input values across the height and width dimensions in atrous convolution.
+ If it is greater than 1, then all values of strides must be 1.
+ name : string
+ A name for this operation (optional).
+
+ Returns
+ -------
+ A 4-D Tensor with shape according to data_format.
+ E.g., for "NHWC" format, shape is [batch, out_height, out_width, in_channels * channel_multiplier].
+ """
+
+ pass
+
+
+class Conv1d_transpose(Cell):
+
+ def __init__(self, strides, padding, data_format, dilations=None, out_channel=None, k_size=None, in_channels=None):
+ super(Conv1d_transpose, self).__init__()
+ self.data_format, self.padding = preprocess_1d_format(data_format, padding)
+ self.in_channels = in_channels
+ self.out_channel = out_channel
+ self.strides = (1, strides)
+ self.dilations = (1, dilations)
+ self.k_size = (1, k_size)
+
+ self.conv2d_transpose = P.Conv2DBackpropInput(
+ out_channel=self.in_channels, kernel_size=self.k_size, pad_mode=self.padding, stride=self.strides,
+ dilation=self.dilations, mode=1, group=1
+ )
+ self.shape = P.Shape()
+ self.expand_dims = P.ExpandDims()
+ self.squeeze = P.Squeeze(2)
+
+ def _deconv_output_length(self, input_length, filter_size, stride_size, dilation_size):
+ length = 0
+ filter_size = filter_size + (filter_size - 1) * (dilation_size - 1)
+
+ if self.padding == 'same':
+ length = input_length * stride_size
+ elif self.padding == 'valid':
+ length = input_length * stride_size + max(filter_size - stride_size, 0)
+
+ return length
+
+ def construct(self, x, filters):
+ if self.data_format == 'NWC':
+ x = nhwc_to_nchw(x)
+ x = self.expand_dims(x, 2)
+ filters = self.expand_dims(filters, 2)
+ n, _, h, w = self.shape(x)
+
+ h_out = self._deconv_output_length(h, self.k_size[0], self.strides[0], self.dilations[0])
+ w_out = self._deconv_output_length(w, self.k_size[1], self.strides[1], self.dilations[1])
+ output = self.conv2d_transpose(x, filters, (n, self.out_channel, h_out, w_out))
+ output = self.squeeze(output)
+
+ if self.data_format == 'NWC':
+ output = nchw_to_nhwc(output)
+ return output
+
+
+def conv1d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NWC', dilations=None, name=None
+):
+ """
+ The transpose of conv1d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 3-D Tensor of type float and shape [batch, in_width, in_channels]
+ for NWC data format or [batch, in_channels, in_width] for NCW data format.
+ filters : tensor
+ A 3-D Tensor with the same type as value and shape [filter_width, output_channels, in_channels].
+ filter's in_channels dimension must match that of value.
+ output_shape : tensor
+ A 1-D Tensor, containing three elements, representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1 or 3. The number of entries by which the filter is moved right at each step.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NWC' and 'NCW' are supported.
+ dilations : list
+ An int or list of ints that has length 1 or 3 which defaults to 1.
+ The dilation factor for each dimension of input. If set to k > 1,
+ there will be k-1 skipped cells between each filter element on that dimension.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+ pass
+
+
+class Conv2d_transpose(Cell):
+
+ def __init__(self, strides, padding, data_format, dilations=None, out_channel=None, k_size=None, in_channels=None):
+ super(Conv2d_transpose, self).__init__()
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.in_channels = in_channels
+ self.out_channel = out_channel
+
+ self.k_size = k_size
+ if self.data_format == 'NHWC':
+ self.strides = (strides[1], strides[2])
+ self.dilations = (dilations[1], dilations[2])
+ elif self.data_format == 'NCHW':
+ self.strides = (strides[2], strides[3])
+ self.dilations = (dilations[2], dilations[3])
+
+ self.conv2d_transpose = P.Conv2DBackpropInput(
+ out_channel=self.in_channels, kernel_size=self.k_size, pad_mode=self.padding, stride=self.strides,
+ dilation=self.dilations, mode=1, group=1
+ )
+ self.shape = P.Shape()
+
+ def _deconv_output_length(self, input_length, filter_size, stride_size, dilation_size):
+ length = 0
+ filter_size = filter_size + (filter_size - 1) * (dilation_size - 1)
+
+ if self.padding == 'same':
+ length = input_length * stride_size
+ elif self.padding == 'valid':
+ length = input_length * stride_size + max(filter_size - stride_size, 0)
+
+ return length
+
+ def construct(self, x, filters):
+ if self.data_format == 'NHWC':
+ x = nhwc_to_nchw(x)
+
+ n, _, h, w = self.shape(x)
+
+ h_out = self._deconv_output_length(h, self.k_size[0], self.strides[0], self.dilations[0])
+ w_out = self._deconv_output_length(w, self.k_size[1], self.strides[1], self.dilations[1])
+
+ output = self.conv2d_transpose(x, filters, (n, self.out_channel, h_out, w_out))
+
+ if self.data_format == 'NHWC':
+ output = nchw_to_nhwc(x)
+
+ return output
+
+
+def conv2d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NHWC', dilations=None, name=None
+):
+ """
+ The transpose of conv2d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 4-D Tensor of type float and shape [batch, height, width, in_channels]
+ for NHWC data format or [batch, in_channels, height, width] for NCHW data format.
+ filters : tensor
+ A 4-D Tensor with the same type as input and shape [height, width,
+ output_channels, in_channels]. filter's in_channels dimension must match that of input.
+ output_shape : tensor
+ A 1-D Tensor representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1, 2 or 4. The stride of the sliding window for each dimension of input.
+ If a single value is given it is replicated in the H and W dimension.
+ By default the N and C dimensions are set to 0.
+ The dimension order is determined by the value of data_format, see below for details.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NHWC' and 'NCHW' are supported.
+ dilations : list
+ An int or list of ints that has length 1, 2 or 4, defaults to 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as input.
+ """
+ pass
+
+
+class Conv3d_transpose(Cell):
+ pass
+
+
+def conv3d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NDHWC', dilations=None, name=None
+):
+ """
+ The transpose of conv3d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of type float and shape [batch, height, width, in_channels] for
+ NHWC data format or [batch, in_channels, height, width] for NCHW data format.
+ filters : tensor
+ A 5-D Tensor with the same type as value and shape [height, width, output_channels, in_channels].
+ filter's in_channels dimension must match that of value.
+ output_shape : tensor
+ A 1-D Tensor representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1, 3 or 5.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NDHWC' and 'NCDHW' are supported.
+ dilations : list of ints
+ An int or list of ints that has length 1, 3 or 5, defaults to 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ pass
+
+
+
+class BatchNorm(Cell):
+ """Batch Normalization base class."""
+
+ @cell_attr_register
+ def __init__(
+ self, num_features, epsilon=1e-5, decay=0.9, gamma=None, beta=None, moving_mean=None, moving_var=None,
+ is_train=None, device_num_each_group=1, data_format='channels_last'
+ ):
+ super(BatchNorm, self).__init__()
+ if num_features < 1:
+ raise ValueError("num_features must be at least 1")
+
+ if decay < 0 or decay > 1:
+ raise ValueError("momentum should be a number in range [0, 1], but got {}".format(decay))
+
+ self.data_format = data_format
+ self.use_batch_statistics = is_train
+ self.num_features = num_features
+ self.eps = epsilon
+ self.moving_mean = moving_mean
+ self.moving_variance = moving_var
+ self.gamma = gamma
+ self.beta = beta
+ self.group = check_int_positive(device_num_each_group)
+ self.is_global = False
+ if self.group != 1:
+ self.rank_id = get_rank()
+ self.rank_size = get_group_size()
+ self.device_list = [i for i in range(0, self.rank_size)]
+ self.rank_list = self.list_group(self.device_list, self.group)
+ self.rank_list_idx = len(self.rank_list)
+ for i in range(self.rank_list_idx):
+ if self.rank_id in self.rank_list[i] and self.group != 1:
+ self.is_global = True
+ management.create_group('group' + str(i), self.rank_list[i])
+ self.all_reduce = P.AllReduce(P.ReduceOp.SUM, 'group' + str(i)).add_prim_attr('fusion', 1)
+ self.shape = P.Shape()
+ self.reduce_mean = P.ReduceMean(keep_dims=True)
+ self.square = P.Square()
+ self.sqrt = P.Sqrt()
+ self.cast = P.Cast()
+ self.dtype = P.DType()
+ self.reshape = P.Reshape()
+ self.is_ascend = context.get_context("device_target") == "Ascend"
+ self.is_gpu = context.get_context("device_target") == "GPU"
+ self.is_graph_mode = context.get_context("mode") == context.GRAPH_MODE
+ self.momentum = 1.0 - decay
+ if context.get_context("enable_ge"):
+ self.is_ge_backend = True
+ else:
+ self.is_ge_backend = False
+
+ if self.is_graph_mode and (self.is_ge_backend or self.is_ascend):
+ self.bn_train = P.BatchNorm(is_training=True, epsilon=self.eps)
+ elif self.is_gpu:
+ self.bn_train = P.FusedBatchNormEx(mode=1, epsilon=self.eps, momentum=self.momentum)
+ else:
+ self.bn_train = P.FusedBatchNorm(mode=1, epsilon=self.eps, momentum=self.momentum)
+ self.bn_infer = P.BatchNorm(is_training=False, epsilon=self.eps)
+ self.enable_global_sync = self.is_global and (self.is_ge_backend or (self.is_graph_mode and self.is_ascend))
+ self.enable_default_train = self.is_graph_mode and not self.is_global and \
+ (self.is_ge_backend or self.is_ascend)
+
+ data_parallel_strategy = ((1, ), (1, ))
+ data_parallel_strategy_one = ((1, ), ())
+ self.sub_mean = P.Sub().shard(data_parallel_strategy)
+ self.sub_var = P.Sub().shard(data_parallel_strategy)
+ self.mul_mean = P.Mul().shard(data_parallel_strategy_one)
+ self.mul_var = P.Mul().shard(data_parallel_strategy_one)
+ self.assign_sub_mean = P.AssignSub().shard(data_parallel_strategy)
+ self.assign_sub_var = P.AssignSub().shard(data_parallel_strategy)
+
+ def _check_data_dim(self, x):
+ raise NotImplementedError
+
+ def list_group(self, world_rank, group_size):
+ if group_size > get_group_size():
+ raise ValueError(
+ "group size can not be greater than local rank size, group size is {}, "
+ "local_rank_size is {}".format(group_size, get_group_size())
+ )
+ if len(world_rank) % group_size != 0:
+ raise ValueError("please make your group size correct.")
+ world_rank_list = zip(*(iter(world_rank), ) * group_size)
+ group_list = [list(i) for i in world_rank_list]
+ return group_list
+
+ def _global_sync(self, x, axes, re_shape):
+ """calculate global batch normalization output"""
+ x_mean = self.reduce_mean(x, axes)
+ x_mean_square = self.reduce_mean(self.square(x), axes)
+ global_batch_mean = self.all_reduce(x_mean) / self.group
+ global_batch_mean_square = self.all_reduce(x_mean_square) / self.group
+ global_mean = global_batch_mean
+ global_var = global_batch_mean_square - self.square(global_mean)
+ var_sqrt = self.sqrt(global_var + self.eps)
+ mean_first = (x - global_mean) / var_sqrt
+ y = mean_first * self.reshape(self.gamma, re_shape) + self.reshape(self.beta, re_shape)
+
+ mean_sub = self.sub_mean(self.reshape(self.moving_mean, re_shape), global_mean)
+ tmp_mean = self.mul_mean(mean_sub, self.cast(self.momentum, self.dtype(mean_sub)))
+ mean_sub2 = self.sub_var(self.reshape(self.moving_mean, re_shape), global_var)
+ tmp_variance = self.mul_var(mean_sub2, self.cast(self.momentum, self.dtype(mean_sub2)))
+ y = F.depend(y, self.assign_sub_mean(self.moving_mean, self.reshape(tmp_mean, self.shape(self.moving_mean))))
+ y = F.depend(
+ y, self.assign_sub_var(self.moving_variance, self.reshape(tmp_variance, self.shape(self.moving_variance)))
+ )
+ return y
+
+ def get_dim(self, input):
+ dim = len(self.shape(input))
+ if dim == 2:
+ return '1d'
+ elif dim == 4:
+ return '2d'
+ else:
+ raise ValueError("The input must has 2 dims or 4 dims.")
+
+ def _shape_check_bn(self, in_shape, in_dims):
+ dim = len(in_shape)
+ if in_dims == '1d' and dim != 2:
+ raise ValueError("The input must has 2 dims.")
+ if in_dims == '2d' and dim != 4:
+ raise ValueError("The input must has 4 dims.")
+ if in_dims == 'both' and dim != 2 and dim != 4:
+ raise ValueError("The input must has 2 dims or 4 dims.")
+
+ def _shape_infer(self, x_shape, num_feature):
+ """global batch normalization shape and axes infer"""
+ if len(x_shape) == 4:
+ axes = (0, 2, 3)
+ re_shape = (1, num_feature, 1, 1)
+ else:
+ axes = (0, )
+ re_shape = (1, num_feature)
+ return axes, re_shape
+
+ def construct(self, inputs):
+ x = inputs
+ self._shape_check_bn(self.shape(x), self.get_dim(x))
+ if self.use_batch_statistics is None:
+ flag = self.training
+ else:
+ flag = self.use_batch_statistics
+
+ if flag:
+ if self.enable_global_sync:
+ if self.data_format == 'channels_last' and self.get_dim(x) == '2d':
+ x = nhwc_to_nchw(x)
+ axes, re_shape = self._shape_infer(F.shape(x), self.num_features)
+ y = self._global_sync(x, axes, re_shape)
+ if self.data_format == 'channels_last' and self.get_dim(x) == '2d':
+ y = nchw_to_nhwc(y)
+ return y
+
+ if self.enable_default_train:
+ if self.data_format == 'channels_last' and self.get_dim(x) == '2d':
+ x = nhwc_to_nchw(x)
+ y, batch_mean, batch_var, _, _ = self.bn_train(x, self.gamma, self.beta, None, None)
+
+ mean_sub = self.sub_mean(self.moving_mean, batch_mean)
+ temp_mean = self.mul_mean(mean_sub, self.momentum)
+ mean_sub2 = self.sub_var(self.moving_variance, batch_var)
+ temp_variance = self.mul_var(mean_sub2, self.momentum)
+ y = F.depend(y, self.assign_sub_mean(self.moving_mean, temp_mean))
+ y = F.depend(y, self.assign_sub_var(self.moving_variance, temp_variance))
+ if self.data_format == 'channels_last' and self.get_dim(x) == '2d':
+ y = nchw_to_nhwc(y)
+ return y
+
+ if self.data_format == 'channels_last' and self.get_dim(x) == '2d':
+ x = nhwc_to_nchw(x)
+ y = self.bn_train(x, self.gamma, self.beta, self.moving_mean, self.moving_variance)[0]
+ if self.data_format == 'channels_last' and self.get_dim(x) == '2d':
+ y = nchw_to_nhwc(y)
+ return y
+ if self.data_format == 'channels_last' and self.get_dim(x) == '2d':
+ x = nhwc_to_nchw(x)
+ y = self.bn_infer(x, self.gamma, self.beta, self.moving_mean, self.moving_variance)[0]
+ if self.data_format == 'channels_last' and self.get_dim(x) == '2d':
+ y = nchw_to_nhwc(y)
+ return y
diff --git a/tensorlayer/backend/ops/paddle_backend.py b/tensorlayer/backend/ops/paddle_backend.py
new file mode 100644
index 0000000..e9b37c5
--- /dev/null
+++ b/tensorlayer/backend/ops/paddle_backend.py
@@ -0,0 +1,972 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, division, print_function
+import paddle as pd
+import paddle.nn as nn
+
+_dtypeDict = ["float16", "float32", "float64", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64"]
+# TODO NotImplemented
+DType = None
+float16 = "float16"
+float32 = "float32"
+float64 = "float64"
+int8 = "int8"
+int16 = "int16"
+int32 = "int32"
+int64 = "int64"
+uint8 = "uint8"
+uint16 = "uint16"
+uint32 = "uint32"
+uint64 = "uint64"
+
+def _getter(init_fn, **kwargs):
+ """Return an named eager tensor."""
+ raise NotImplementedError
+
+
+def set_context(**kwargs):
+ raise Exception("Using Paddle backend,You don't need to set context")
+
+
+def get_tensor_shape(x):
+ return pd.shape(x)
+
+
+# initializers
+def zeros(shape, dtype="float32"):
+ """
+ Creates a tensor with all elements set to zero.
+
+ Parameters
+ ----------
+ shape : A list of integers
+ a tuple of integers, or a 1-D Tensor of type int32.
+ dtype : tensor
+ The DType of an element in the resulting Tensor
+
+ Returns
+ -------
+ A Tensor with all elements set to zero.
+
+ """
+ return pd.zeros(shape=shape, dtype=dtype)
+
+
+def ones(shape, dtype="float32"):
+ """
+ Creates a tensor with all elements set to ones.
+
+ Parameters
+ ----------
+ shape : A list of integers
+ a tuple of integers, or a 1-D Tensor of type int32.
+ dtype : tensor
+ The DType of an element in the resulting Tensor
+
+ Returns
+ -------
+ A Tensor with all elements set to zero.
+
+ """
+ return pd.ones(shape=shape, dtype=dtype)
+
+
+def constant(value, shape, dtype="float32"):
+ """
+ Creates a constant tensor from a tensor-like object.
+
+ Parameters
+ ----------
+ value : list
+ A constant value (or list) of output type dtype.
+ dtype : tensor
+ The type of the elements of the resulting tensor.
+ shape : tuple
+ Optional dimensions of resulting tensor.
+
+ Returns
+ -------
+ A Constant Tensor.
+
+ """
+ return nn.initializer.constant(value=value)
+
+
+def random_uniform(shape, minval=0, maxval=None, dtype="float32", seed=None):
+ """
+ Outputs random values from a uniform distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ minval : int
+ The lower bound on the range of random values to generate (inclusive). Defaults to 0.
+ maxval : int
+ The upper bound on the range of random values to generate (exclusive). Defaults to 1 if dtype is floating point.
+ dtype : tensor
+ The type of the output: float16, float32, float64, int32, or int64.
+ seed : int
+ Used in combination with dragon.random.set_seed to create a reproducible sequence of tensors across multiple calls.
+ Returns
+ -------
+ A tensor of the specified shape filled with random uniform values.
+
+ """
+ raise NotImplementedError
+
+
+def random_normal(shape, mean=0.0, stddev=1.0, dtype="float32", seed=None):
+ """
+ Outputs random values from a normal distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ dtype : tensor
+ The type of the output.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns
+ -------
+ A tensor of the specified shape filled with random normal values.
+
+ """
+ raise NotImplementedError
+
+
+def truncated_normal(shape, mean=0.0, stddev=1.0, dtype="float32", seed=None):
+ """
+ Outputs random values from a truncated normal distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ dtype : tensor
+ The type of the output.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns
+ -------
+ A tensor of the specified shape filled with random truncated normal values.
+
+ """
+ raise NotImplementedError
+
+
+def he_normal(shape, dtype, seed=None):
+ """
+ He normal initializer.
+
+ Parameters
+ ----------
+ seed : A Python integer.
+ Used to seed the random generator.
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ dtype : tensor
+ The type of the output.
+
+ Returns
+ -------
+ A tensor of the specified shape filled with he normal values.
+ """
+ # shape = shape[::-1]
+ raise NotImplementedError
+
+
+def Variable(initial_value, name, trainable=None):
+ """
+ Creates a new variable with value initial_value.
+
+ Parameters
+ ----------
+ initial_value : tensor
+ A Tensor, or Python object convertible to a Tensor
+ name : str
+ Optional name for the variable. Defaults to 'Variable' and gets uniquified automatically.
+ Returns
+ -------
+ Variable
+ """
+ raise NotImplementedError
+
+
+class MatMul(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, a, b):
+ return pd.matmul(x=a, y=b)
+
+
+def matmul(a, b):
+ """
+ Multiplies matrix a by matrix b, producing a * b.
+
+ Parameters
+ ----------
+ a : tensor
+ type float16, float32, float64, int32, complex64, complex128 and rank > 1.
+ b : tensor
+ with same type and rank as a.
+
+ Returns
+ -------
+ A Tensor of the same type as a and b
+ """
+ raise NotImplementedError
+
+
+def add(value, bias):
+ """
+ Returns x + y element-wise.
+
+ Parameters
+ ----------
+ value : tensor.
+ Must be one of the following types: bfloat16, half, float32, float64,
+ uint8, int8, int16, int32, int64, complex64, complex128, string.
+ bias : tensor
+ Must have the same type as a
+ name : str
+ A name for the operation
+
+ Returns
+ -------
+ A Tensor. Has the same type as a.
+ """
+
+ raise NotImplementedError
+
+
+def dtypes(dt):
+ """
+ Data dtypes.
+
+ Parameters
+ ----------
+ dt : string
+ It could be 'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16',
+ 'int32', 'int64', 'float16', 'float32', 'float64', 'DType'.
+
+ Returns
+ -------
+ Data dtypes
+ """
+ raise NotImplementedError
+
+
+class Maximum(object):
+ def __init__(self):
+ pass
+
+ def __call__(self, x, y):
+ raise NotImplementedError
+
+
+class Minimum(object):
+ def __init__(self):
+ pass
+
+ def __call__(self, x, y):
+ raise NotImplementedError
+
+
+def minimum(x, y):
+ """
+ Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+
+ Parameters
+ ----------
+ x : tensor.
+ Must be one of the following types: bfloat16, half, float32, float64, int32, int64.
+ y : A Tensor.
+ Must have the same type as x.
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor. Has the same type as x
+ """
+ raise NotImplementedError
+
+
+class FlattenReshape(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, inputs):
+ return pd.flatten(x=inputs, start_axis=1,stop_axis=-1)
+
+
+class Reshape(object):
+
+ def __init__(self, shape):
+ self.shape = shape
+
+ def __call__(self, tensor):
+ raise NotImplementedError
+
+
+def reshape(tensor, shape):
+ """
+ Reshapes a tensor.
+
+ Parameters
+ ----------
+ tensor : tensor
+ A Tensor.
+ shape : tensor
+ Defines the shape of the output tensor.
+ Returns
+ -------
+ A Tensor. Has the same type as tensor
+ """
+ raise NotImplementedError
+
+
+class Concat(object):
+
+ def __init__(self, axis):
+ super(Concat, self).__init__()
+ self.axis = axis
+
+ def __call__(self, values):
+ raise NotImplementedError
+
+
+def concat(values, axis):
+ """
+ Concatenates tensors along one dimension.
+
+ Parameters
+ ----------
+ values : list
+ A list of Tensor objects or a single Tensor
+ axis : int
+ 0-D int32 Tensor. Dimension along which to concatenate
+ Returns
+ -------
+ A Tensor resulting from concatenation of the input tensors.
+ """
+ raise NotImplementedError
+
+
+def convert_to_tensor(value, dtype=None):
+ """
+ Converts the given value to a Tensor.
+
+ Parameters
+ ----------
+ value : object
+ An object whose type has a registered Tensor conversion function.
+ dtype : optional
+ Optional element type for the returned tensor. If missing, the type is inferred from the type of value.
+
+ Returns
+ -------
+ A Tensor based on value.
+ """
+ raise NotImplementedError
+
+
+def sqrt(x):
+ """
+ Computes square root of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x.
+ """
+ raise NotImplementedError
+
+
+class ReduceSum(object):
+
+ def __init__(self, axis):
+ pass
+
+ def construct(self, input):
+ pass
+
+
+class ReduceMean(object):
+
+ def __init__(self, axis):
+ if axis == [1, 2]:
+ self.data_format = 'NHWC'
+ elif axis == [2, 3]:
+ self.data_format = 'NCHW'
+ else:
+ raise ("`data_format` should have one of the following values: [`channels_last`, `channels_first`]")
+
+ def __call__(self, inputs):
+ raise NotImplementedError
+
+
+def reduce_mean(input_tensor, axis=None):
+ """
+ Computes the mean of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ raise NotImplementedError
+
+
+class ReduceMax(object):
+
+ def __init__(self, axis):
+ if axis == [1, 2]:
+ self.data_format = 'NHWC'
+ elif axis == [2, 3]:
+ self.data_format = 'NCHW'
+ else:
+ raise ("`data_format` should have one of the following values: [`channels_last`, `channels_first`]")
+
+ def __call__(self, inputs):
+ raise NotImplementedError
+
+
+def reduce_max(input_tensor, axis=None):
+ """
+ Computes the maximum of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have real numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ raise NotImplementedError
+
+
+def reduce_min(input_tensor, axis=None):
+ """
+ Computes the minimum of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have real numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+ raise NotImplementedError
+
+class Pad(object):
+ def __init__(self, paddings, mode="REFLECT"):
+ if mode not in ['CONSTANT', 'REFLECT', 'SYMMETRIC']:
+ raise Exception("Unsupported mode: {}".format(mode))
+ if mode == 'SYMMETRIC':
+ mode = 'EDGE'
+ self.paddings = paddings
+ self.mode = mode
+
+ def __call__(self, x):
+ raise NotImplementedError
+
+def pad(tensor, paddings, mode='CONSTANT', constant_values=0):
+ """
+ Pads a tensor.
+
+ Parameters
+ ----------
+ tensor : tensor
+ A Tensor.
+ paddings : tuple
+ A tuple of type int32.
+ mode : str
+ One of "CONSTANT", "REFLECT", or "SYMMETRIC" (case-insensitive)
+ constant_values : int
+ In "CONSTANT" mode, the scalar pad value to use. Must be same type as tensor.
+
+ Returns
+ -------
+ A Tensor. Has the same type as tensor.
+ """
+ raise NotImplementedError
+
+
+class Unstack(object):
+
+ def __init__(self, axis, num=None):
+ self.axis = axis
+ self.num = num
+
+ def __call__(self, values):
+ raise NotImplementedError
+
+
+class Stack(object):
+
+ def __init__(self, axis):
+ self.axis = axis
+
+ def __call__(self, values):
+ raise NotImplementedError
+
+
+def stack(values, axis=0):
+ """
+ Stacks a list of rank-R tensors into one rank-(R+1) tensor.
+
+ Parameters
+ ----------
+ values : list
+ A list of Tensor objects with the same shape and type.
+ axis : int
+ An int. The axis to stack along. Defaults to the first dimension.
+ Negative values wrap around, so the valid range is [-(R+1), R+1).
+
+ Returns
+ -------
+ A stacked Tensor with the same type as values.
+ """
+ raise NotImplementedError
+
+
+class Meshgrid(object):
+ def __init__(self, indexing='xy'):
+ super(Meshgrid, self).__init__()
+ self.index = indexing
+
+ def __call__(self, inputs):
+ pass
+
+
+def meshgrid(x, y):
+ """
+ Broadcasts parameters for evaluation on an N-D grid.
+
+ Parameters
+ ----------
+ x : tensor
+ Tensors with rank 1.
+ y : tensor
+ Tensors with rank 1.
+
+ Returns
+ -------
+ A list of N Tensors with rank N.
+ """
+
+ pass
+
+
+def range(start, limit=None, delta=1, dtype=None):
+ """
+ Creates a sequence of numbers.
+
+ Parameters
+ ----------
+ start : tensor
+ A 0-D Tensor (scalar). Acts as first entry in the range if limit is not None;
+ otherwise, acts as range limit and first entry defaults to 0.
+ limit : tensor
+ A 0-D Tensor (scalar). Upper limit of sequence, exclusive. If None,
+ defaults to the value of start while the first entry of the range defaults to 0.
+ delta : tensor
+ A 0-D Tensor (scalar). Number that increments start. Defaults to 1.
+ dtype : type
+ The type of the elements of the resulting tensor.
+
+ Returns
+ -------
+ An 1-D Tensor of type dtype.
+ """
+ raise NotImplementedError
+
+
+class ExpandDims(object):
+
+ def __init__(self, axis):
+ pass
+
+ def construct(self, input):
+ pass
+
+
+def expand_dims(input, axis):
+ """
+ Inserts a dimension of 1 into a tensor's shape.
+
+ Parameters
+ ----------
+ input : tensor
+ A Tensor.
+ axis : int
+ 0-D (scalar). Specifies the dimension index at which to expand the shape of input.
+ Must be in the range [-rank(input) - 1, rank(input)].
+
+ Returns
+ -------
+ A Tensor with the same data as input, but its shape has an additional dimension of size 1 added.
+ """
+
+ raise NotImplementedError
+
+
+class Tile(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, input, multiples):
+ raise NotImplementedError
+
+
+def tile(input, multiples):
+ """
+ Constructs a tensor by tiling a given tensor.
+
+ Parameters
+ ----------
+ input : tensor
+ A Tensor. 1-D or higher.
+ multiples : tensor
+ Must be one of the following types: int32, int64. 1-D.
+ Length must be the same as the number of dimensions in input
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ raise NotImplementedError
+
+
+class Cast(object):
+
+ def __init__(self, dtype):
+ pass
+
+ def __call__(self, input):
+ pass
+
+
+def cast(x, dtype):
+ """
+ Casts a tensor to a new type.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor or SparseTensor or IndexedSlices of numeric type.
+ It could be uint8, uint16, uint32, uint64, int8, int16, int32, int64, float16, float32, float64.
+ dtype : dtpye
+ The destination type. The list of supported dtypes is the same as x
+
+ Returns
+ -------
+ A Tensor or SparseTensor or IndexedSlices with same shape as x and same type as dtype.
+ """
+ raise NotImplementedError
+
+
+class Transpose(object):
+
+ def __init__(self, perm, conjugate=False):
+ self.perm = perm
+ if conjugate:
+ raise ("The conjugate Parameters not supported")
+
+ def __call__(self, a):
+ raise NotImplementedError
+
+
+def transpose(a, perm=None, conjugate=False):
+ """
+ Transposes a.
+
+ Parameters
+ ----------
+ a : tensor
+ A Tensor.
+ perm : int
+ A permutation of the dimensions of a.
+ conjugate : bool
+ Setting it to True is mathematically equivalent to ms.math.conj(ms.transpose(input)).
+
+ Returns
+ -------
+ A transposed Tensor.
+ """
+
+ raise NotImplementedError
+
+
+def gather_nd(params, indices, batch_dims=0):
+ """
+ Gather slices from params into a Tensor with shape specified by indices.
+
+ Parameters
+ ----------
+ params : tensor
+ The tensor from which to gather values.
+ indices : tensor
+ Must be one of the following types: int32, int64. Index tensor.
+ batch_dims : int
+ An integer or a scalar 'Tensor'. The number of batch dimensions.
+
+ Returns
+ -------
+ A Tensor. Has the same type as params.
+ """
+
+ pass
+
+
+def clip_by_value(t, clip_value_min, clip_value_max):
+ """
+ Clips tensor values to a specified min and max.
+
+ Parameters
+ ----------
+ t : tensor
+ A Tensor or IndexedSlices
+ clip_value_min : tensor
+ A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by
+ clip_value_max : tensor
+ A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by
+
+ Returns
+ -------
+ A clipped Tensor or IndexedSlices.
+ """
+
+ pass
+
+
+def split(value, num_or_size_splits, axis=0, num=None):
+ """
+ Splits a tensor into sub tensors.
+
+ Parameters
+ ----------
+ value : tensor
+ The Tensor to split.
+ num_or_size_splits : list
+ Either an integer indicating the number of splits along split_dim or a 1-D integer Tensor or
+ Python list containing the sizes of each output tensor along split_dim.
+ axis : int
+ The dimension along which to split. Must be in the range [-rank(value), rank(value)). Defaults to 0.
+ num : int
+ used to specify the number of outputs when it cannot be inferred from the shape of size_splits.
+
+ Returns
+ -------
+ Tensor objects resulting from splitting value.
+ """
+ pass
+
+
+def floor(x):
+ raise NotImplementedError
+
+
+def gather(params, indices):
+ raise NotImplementedError
+
+
+def linspace(start, stop, num):
+ raise NotImplementedError
+
+
+def slice(inputs, starts, sizes):
+ raise NotImplementedError
+
+
+def add_n(inputs):
+ raise NotImplementedError
+
+
+class OneHot(object):
+
+ def __init__(self, axis=-1, depth=1, on_value=1.0, off_value=0.0, dtype="float32"):
+ self.depth = depth
+ self.dtype = dtype
+
+ def __call__(self, indices):
+ raise NotImplementedError
+
+
+class L2Normalize(object):
+
+ def __init__(self, axis=None, epsilon=1e-12):
+ super(L2Normalize, self).__init__()
+ pass
+
+ def __call__(self, input, *args, **kwargs):
+ pass
+
+
+class EmbeddingLookup(object):
+
+ def __init__(self, max_norm=None):
+ self.max_norm = max_norm
+
+ def __call__(self, params, ids, *args, **kwargs):
+ pass
+
+
+class NCELoss(object):
+
+ def __init__(self, num_true=1, sampled_values=None, remove_accidental_hits=False):
+ super(NCELoss, self).__init__()
+
+ def __call__(self, weights, biases, labels, inputs, num_sampled, num_classes):
+ pass
+
+
+class Not_equal(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x, y):
+ pass
+
+
+class Count_nonzero(object):
+
+ def __init__(self, keepdims=None, dtype="int64"):
+ pass
+
+ def __call__(self, *args, **kwargs):
+ pass
+
+
+
+class Resize:
+
+ def __init__(self, scale, method, antialias=False, data_format='channels_last', ksize=None):
+ if method not in ['nearest', 'linear', 'bilinear']:
+ raise ('Current resize does not support this method.')
+ if method == 'bilinear':
+ method = 'linear'
+ self.method = method
+ self.antialias = antialias
+ self.scale = scale
+ if data_format != 'channel_last':
+ raise Exception("UpSampling2d resize_images only support channel_last")
+
+ def __call__(self, inputs):
+ raise NotImplementedError
+
+
+def resize(inputs, output_size, method, antialias):
+ raise NotImplementedError
+
+
+class ZeroPadding1D(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, padding):
+ raise NotImplementedError
+
+
+class ZeroPadding2D(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, padding):
+ raise NotImplementedError
+
+
+class ZeroPadding3D(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, padding):
+ raise NotImplementedError
+
+
+class Sign(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ raise NotImplementedError
+
+def ceil(x):
+ raise NotImplementedError
+
+def multiply(x, y):
+ raise NotImplementedError
+
+def divide(x, y):
+ raise NotImplementedError
+
+def identity(x):
+ raise NotImplementedError
+
+class BatchToSpace(object):
+ def __init__(self, block_size, crops):
+ super(BatchToSpace, self).__init__()
+ pass
+
+ def __call__(self, input_x):
+ raise NotImplementedError
+
+
+class DepthToSpace(object):
+ def __init__(self, block_size, data_format='NHWC'):
+ pass
+
+ def __call__(self, input):
+ raise NotImplementedError
\ No newline at end of file
diff --git a/tensorlayer/backend/ops/paddle_nn.py b/tensorlayer/backend/ops/paddle_nn.py
new file mode 100644
index 0000000..47d9dd0
--- /dev/null
+++ b/tensorlayer/backend/ops/paddle_nn.py
@@ -0,0 +1,926 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import paddle as pd
+import paddle.nn.functional as F
+
+def padding_format(padding):
+ """
+ Checks that the padding format correspond format.
+
+ Parameters
+ ----------
+ padding : str
+ Must be one of the following:"same", "SAME", "VALID", "valid"
+
+ Returns
+ -------
+ str "SAME" or "VALID"
+ """
+
+ if padding in ["SAME", "same"]:
+ padding = "SAME"
+ elif padding in ["VALID", "valid"]:
+ padding = "VALID"
+ elif padding == None:
+ padding = None
+ else:
+ raise Exception("Unsupported padding: " + str(padding))
+ return padding
+
+
+def preprocess_1d_format(data_format, padding):
+ """
+ Checks that the 1-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NWC","NCW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NWC" or "NCW" and "SAME" or "VALID"
+ """
+
+ if data_format in ["channels_last", "NWC"]:
+ data_format = "NWC"
+ elif data_format in ["channels_first", "NCW"]:
+ data_format = "NCW"
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def preprocess_2d_format(data_format, padding):
+ """
+ Checks that the 2-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NHWC","NCHW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NHWC" or "NCHW" and "SAME" or "VALID"
+ """
+
+ if data_format in ["channels_last", "NHWC", "nhwc"]:
+ data_format = "NHWC"
+ elif data_format in ["channels_first", "NCHW", "nchw"]:
+ data_format = "NCHW"
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def preprocess_3d_format(data_format, padding):
+ """
+ Checks that the 3-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NDHWC","NCDHW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NDHWC" or "NCDHW" and "SAME" or "VALID"
+ """
+
+ if data_format in ['channels_last', 'NDHWC']:
+ data_format = 'NDHWC'
+ elif data_format in ['channels_first', 'NCDHW']:
+ data_format = 'NCDHW'
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def nchw_to_nhwc(x):
+ """
+ Channels first to channels last
+
+ Parameters
+ ----------
+ x : tensor
+ channels first tensor data
+
+ Returns
+ -------
+ channels last tensor data
+ """
+
+ pass
+
+
+def nhwc_to_nchw(x):
+ """
+ Channles last to channels first
+
+ Parameters
+ ----------
+ x : tensor
+ channels last tensor data
+
+ Returns
+ -------
+ channels first tensor data
+ """
+
+ pass
+
+
+class ReLU(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return F.relu(x)
+
+
+def relu(x):
+ """
+ Computes rectified linear: max(features, 0).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: float32, float64, int32, uint8, int16,
+ int8, int64, bfloat16, uint16, half, uint32, uint64, qint8.
+
+ Returns
+ -------
+ A Tensor. Has the same type as features.
+ """
+ return F.relu(x)
+
+
+class ReLU6(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return F.relu6(x)
+
+
+def relu6(x):
+ """
+ Computes Rectified Linear 6: min(max(features, 0), 6).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: float32, float64, int32, uint8, int16,
+ int8, int64, bfloat16, uint16, half, uint32, uint64, qint8.
+
+ Returns
+ -------
+ A Tensor with the same type as features.
+ """
+ return F.relu6(x)
+
+
+class LeakyReLU(object):
+
+ def __init__(self, alpha=0.2):
+ self.alpha = alpha
+
+ def __call__(self, x):
+ return F.leaky_relu(x, negative_slope=self.alpha)
+
+
+def leaky_relu(x):
+ """
+ Compute the Leaky ReLU activation function.
+
+ Parameters
+ ----------
+ x : tensor
+ representing preactivation values. Must be one of the following types:
+ float16, float32, float64, int32, int64.
+
+ Returns
+ -------
+ The activation value.
+ """
+
+ return F.leaky_relu(x)
+
+
+class Softplus(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return F.softplus(x)
+
+
+def softplus(x):
+ """
+ Computes softplus: log(exp(features) + 1).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: half, bfloat16, float32, float64.
+
+ Returns
+ -------
+ A Tensor. Has the same type as features.
+ """
+
+ return F.softplus(x)
+
+
+class Tanh(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return F.tanh(x)
+
+
+def tanh(x):
+ """
+ Computes hyperbolic tangent of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x.
+ """
+
+ return F.tanh(x)
+
+
+class Sigmoid(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return F.sigmoid(x)
+
+
+def sigmoid(x):
+ """
+ Computes sigmoid of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float16, float32, float64, complex64, or complex128.
+
+ Returns
+ -------
+ A Tensor with the same type as x.
+ """
+ return F.sigmoid(x)
+
+
+class Softmax(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return F.softmax(x)
+
+
+def softmax(logits, axis=-1):
+ """
+ Computes softmax activations.
+
+ Parameters
+ ----------
+ logits : tensor
+ Must be one of the following types: half, float32, float64.
+ axis : int
+ The dimension softmax would be performed on. The default is -1 which indicates the last dimension.
+
+ Returns
+ -------
+ A Tensor. Has the same type and shape as logits.
+ """
+ return F.softmax(logits, axis=axis)
+
+
+class Dropout(object):
+
+ def __init__(self, keep, seed=1):
+ self.keep = 1 - keep
+ self.seed = seed
+
+ def __call__(self, inputs):
+ raise NotImplementedError
+
+
+class BiasAdd(object):
+ """
+ Adds bias to value.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float, double, int64, int32, uint8, int16, int8, complex64, or complex128.
+ bias : tensor
+ Must be the same type as value unless value is a quantized type,
+ in which case a different quantized type may be used.
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ def __init__(self, data_format='NHWC'):
+ self.data_format = data_format
+
+ def __call__(self, x, bias):
+ return pd.add(x, bias)
+
+
+def bias_add(x, bias):
+ """
+ Adds bias to value.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float, double, int64, int32, uint8, int16, int8, complex64, or complex128.
+ bias : tensor
+ Must be the same type as value unless value is a quantized type,
+ in which case a different quantized type may be used.
+ data_format : A string.
+ 'N...C' and 'NC...' are supported.
+ name : str
+ A name for the operation (optional).
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+ raise NotImplementedError
+
+
+class Conv1D(object):
+ pass
+ # raise NotImplementedError
+
+
+def conv1d(input, filters, stride, padding, data_format='NWC', dilations=None, name=None):
+ """
+ Computes a 1-D convolution given 3-D input and filter tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ A 3D Tensor. Must be of type float16, float32, or float64
+ filters : tensor
+ A 3D Tensor. Must have the same type as input.
+ stride : int of list
+ An int or list of ints that has length 1 or 3. The number of entries by which the filter is moved right at each step.
+ padding : string
+ 'SAME' or 'VALID'
+ data_format : string
+ An optional string from "NWC", "NCW". Defaults to "NWC", the data is stored in the order of
+ [batch, in_width, in_channels]. The "NCW" format stores data as [batch, in_channels, in_width].
+ dilations : int or list
+ An int or list of ints that has length 1 or 3 which defaults to 1.
+ The dilation factor for each dimension of input. If set to k > 1,
+ there will be k-1 skipped cells between each filter element on that dimension.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ A name for the operation (optional).
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ pass
+
+
+class Conv2D(object):
+
+ def __init__(self, strides, padding, data_format='NHWC', dilations=None, out_channel=None, k_size=None):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.ksize = k_size[0]
+ if self.data_format is 'NHWC':
+ self.dg_stride = strides[1]
+ self.dg_dilation = dilations[1]
+ elif self.data_format is 'NCHW':
+ self.dg_stride = strides[2]
+ self.dg_dilation = dilations[2]
+
+ def __call__(self, inputs, filters):
+ raise NotImplementedError
+
+
+def conv2d(input, filters, strides, padding, data_format='NCHW', dilations=None):
+ """
+ Computes a 2-D convolution given 4-D input and filters tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ Must be one of the following types: half, bfloat16, float32, float64. A 4-D tensor.
+ The dimension order is interpreted according to the value of data_format, see below for details.
+ filters : tensor
+ Must have the same type as input. A 4-D tensor of shape [filter_height, filter_width, in_channels, out_channels]
+ strides : int of list
+ The stride of the sliding window for each dimension of input. If a single value is given it is replicated in the H and W dimension.
+ By default the N and C dimensions are set to 1. The dimension order is determined by the value of data_format, see below for details.
+ padding : string
+ "SAME" or "VALID"
+ data_format : string
+ "NHWC", "NCHW". Defaults to "NCHW".
+ dilations : list or ints
+ list of ints that has length 1, 2 or 4, defaults to 1. The dilation factor for each dimension ofinput.
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ raise NotImplementedError
+
+
+class Conv3D(object):
+ pass
+ # raise NotImplementedError
+
+
+def conv3d(input, filters, strides, padding, data_format='NDHWC', dilations=None, name=None):
+ """
+ Computes a 3-D convolution given 5-D input and filters tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ Must be one of the following types: half, bfloat16, float32, float64.
+ Shape [batch, in_depth, in_height, in_width, in_channels].
+ filters : tensor
+ Must have the same type as input. Shape [filter_depth, filter_height, filter_width, in_channels, out_channels].
+ in_channels must match between input and filters.
+ strides : list of ints
+ A list of ints that has length >= 5. 1-D tensor of length 5.
+ The stride of the sliding window for each dimension of input.
+ Must have strides[0] = strides[4] = 1.
+ padding : string
+ A string from: "SAME", "VALID". The type of padding algorithm to use.
+ data_format : string
+ An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". The data format of the input and output data.
+ With the default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels].
+ Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width].
+ dilations : list of ints
+ Defaults to [1, 1, 1, 1, 1]. 1-D tensor of length 5. The dilation factor for each dimension of input.
+ If set to k > 1, there will be k-1 skipped cells between each filter element on that dimension.
+ The dimension order is determined by the value of data_format, see above for details.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ raise NotImplementedError
+
+
+def lrn(inputs, depth_radius, bias, alpha, beta):
+ """
+ Local Response Normalization.
+
+ Parameters
+ ----------
+ inputs : tensor
+ Must be one of the following types: half, bfloat16, float32. 4-D.
+ depth_radius : int
+ Defaults to 5. 0-D. Half-width of the 1-D normalization window.
+ bias : float
+ Defaults to 1. An offset (usually positive to avoid dividing by 0).
+ alpha : float
+ Defaults to 1. A scale factor, usually positive.
+ beta : float
+ Defaults to 0.5. An exponent.
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+ pass
+
+
+def moments(x, axes, shift=None, keepdims=False):
+ """
+ Calculates the mean and variance of x.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor
+ axes : ints
+ Axes along which to compute mean and variance.
+ shift : int
+ Not used in the current implementation.
+ keepdims : bool
+ produce moments with the same dimensionality as the input.
+
+ Returns
+ -------
+ Two Tensor objects: mean and variance.
+ """
+
+ pass
+
+
+class MaxPool(object):
+
+ def __init__(self, ksize, strides, padding, data_format=None):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.ksize = ksize
+ self.strides = strides
+
+ def __call__(self, inputs):
+ raise NotImplementedError
+
+
+def max_pool(input, ksize, strides, padding, data_format=None):
+ """
+ Performs the max pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels] if data_format does not start
+ with "NC" (default), or [batch_size, num_channels] + input_spatial_shape if data_format starts with "NC".
+ Pooling happens over the spatial dimensions only.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The max pooled output tensor.
+ """
+ pass
+
+
+class AvgPool(object):
+
+ def __init__(self, ksize, strides, padding, data_format=None):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.filter_size = ksize
+ self.strides = strides
+
+ def __call__(self, inputs):
+ raise NotImplementedError
+
+
+def avg_pool(input, ksize, strides, padding):
+ """
+ Performs the avg pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels]
+ if data_format does not start with "NC" (default), or [batch_size, num_channels] + input_spatial_shape
+ if data_format starts with "NC". Pooling happens over the spatial dimensions only.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The average pooled output tensor.
+ """
+ pass
+
+
+def max_pool3d(input, ksize, strides, padding, data_format=None, name=None):
+ """
+ Performs the max pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of the format specified by data_format.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ "NDHWC", "NCDHW". Defaults to "NDHWC". The data format of the input and output data.
+ With the default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels].
+ Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width].
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The max pooled output tensor.
+ """
+ pass
+
+
+def avg_pool3d(input, ksize, strides, padding, data_format=None, name=None):
+ """
+ Performs the average pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of shape [batch, height, width, channels] and type float32, float64, qint8, quint8, or qint32.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, 3 or 5. The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NDHWC' and 'NCDHW' are supported.
+ name : string
+ Optional name for the operation.
+
+ Returns
+ -------
+ A Tensor with the same type as value. The average pooled output tensor.
+ """
+ pass
+
+
+def pool(input, window_shape, pooling_type, strides=None, padding='VALID', data_format=None, dilations=None, name=None):
+ """
+ Performs an N-D pooling operation.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels]
+ if data_format does not start with "NC" (default), or [batch_size, num_channels] + input_spatial_shape
+ if data_format starts with "NC". Pooling happens over the spatial dimensions only.
+ window_shape : int
+ Sequence of N ints >= 1.
+ pooling_type : string
+ Specifies pooling operation, must be "AVG" or "MAX".
+ strides : ints
+ Sequence of N ints >= 1. Defaults to [1]*N. If any value of strides is > 1, then all values of dilation_rate must be 1.
+ padding : string
+ The padding algorithm, must be "SAME" or "VALID". Defaults to "SAME".
+ See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ Specifies whether the channel dimension of the input and output is the last dimension (default, or if data_format does not start with "NC"),
+ or the second dimension (if data_format starts with "NC").
+ For N=1, the valid values are "NWC" (default) and "NCW". For N=2, the valid values are "NHWC" (default) and "NCHW".
+ For N=3, the valid values are "NDHWC" (default) and "NCDHW".
+ dilations : list of ints
+ Dilation rate. List of N ints >= 1. Defaults to [1]*N. If any value of dilation_rate is > 1, then all values of strides must be 1.
+ name : string
+ Optional. Name of the op.
+
+ Returns
+ -------
+ Tensor of rank N+2, of shape [batch_size] + output_spatial_shape + [num_channels]
+ """
+ pass
+
+
+class DepthwiseConv2d(object):
+
+ def __init__(self, strides, padding, data_format=None, dilations=None, ksize=None, channel_multiplier=1):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.stride = strides
+ self.dilations = dilations
+
+ def __call__(self, input, filter):
+ raise NotImplementedError("Not implemented depthwiseconv2d")
+
+
+def depthwise_conv2d(input, filter, strides, padding, data_format=None, dilations=None, name=None):
+ """
+ Depthwise 2-D convolution.
+
+ Parameters
+ ----------
+ input : tensor
+ 4-D with shape according to data_format.
+ filter : tensor
+ 4-D with shape [filter_height, filter_width, in_channels, channel_multiplier].
+ strides : list
+ 1-D of size 4. The stride of the sliding window for each dimension of input.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ The data format for input. Either "NHWC" (default) or "NCHW".
+ dilations : list
+ 1-D of size 2. The dilation rate in which we sample input values across the height and width dimensions in atrous convolution.
+ If it is greater than 1, then all values of strides must be 1.
+ name : string
+ A name for this operation (optional).
+
+ Returns
+ -------
+ A 4-D Tensor with shape according to data_format.
+ E.g., for "NHWC" format, shape is [batch, out_height, out_width, in_channels * channel_multiplier].
+ """
+
+ pass
+
+class Conv1d_transpose(object):
+
+ def __init__(
+ self, strides, padding, data_format='NWC', dilations=None, out_channel=None, k_size=None, in_channels=None
+ ):
+ self.strides = strides
+ self.dilations = dilations
+ self.data_format, self.padding = preprocess_1d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ raise NotImplementedError
+
+
+def conv1d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NWC', dilations=None, name=None
+):
+ """
+ The transpose of conv1d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 3-D Tensor of type float and shape [batch, in_width, in_channels]
+ for NWC data format or [batch, in_channels, in_width] for NCW data format.
+ filters : tensor
+ A 3-D Tensor with the same type as value and shape [filter_width, output_channels, in_channels].
+ filter's in_channels dimension must match that of value.
+ output_shape : tensor
+ A 1-D Tensor, containing three elements, representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1 or 3. The number of entries by which the filter is moved right at each step.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NWC' and 'NCW' are supported.
+ dilations : list
+ An int or list of ints that has length 1 or 3 which defaults to 1.
+ The dilation factor for each dimension of input. If set to k > 1,
+ there will be k-1 skipped cells between each filter element on that dimension.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+ pass
+
+
+class Conv2d_transpose(object):
+
+ def __init__(
+ self, strides, padding, data_format='NHWC', dilations=None, name=None, out_channel=None, k_size=None,
+ in_channels=None
+ ):
+ self.strides = strides
+ self.dilations = dilations
+ self.name = name
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ raise NotImplementedError
+
+
+def conv2d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NHWC', dilations=None, name=None
+):
+ """
+ The transpose of conv2d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 4-D Tensor of type float and shape [batch, height, width, in_channels]
+ for NHWC data format or [batch, in_channels, height, width] for NCHW data format.
+ filters : tensor
+ A 4-D Tensor with the same type as input and shape [height, width,
+ output_channels, in_channels]. filter's in_channels dimension must match that of input.
+ output_shape : tensor
+ A 1-D Tensor representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1, 2 or 4. The stride of the sliding window for each dimension of input.
+ If a single value is given it is replicated in the H and W dimension.
+ By default the N and C dimensions are set to 0.
+ The dimension order is determined by the value of data_format, see below for details.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NHWC' and 'NCHW' are supported.
+ dilations : list
+ An int or list of ints that has length 1, 2 or 4, defaults to 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as input.
+ """
+ pass
+
+
+class Conv3d_transpose(object):
+
+ def __init__(
+ self, strides, padding, data_format='NDHWC', dilations=None, name=None, out_channel=None, k_size=None,
+ in_channels=None
+ ):
+ self.strides = strides
+ self.dilations = dilations
+ self.name = name
+ self.out_channel = out_channel
+ self.data_format, self.padding = preprocess_3d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ raise NotImplementedError
+
+
+def conv3d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NDHWC', dilations=None, name=None
+):
+ """
+ The transpose of conv3d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of type float and shape [batch, height, width, in_channels] for
+ NHWC data format or [batch, in_channels, height, width] for NCHW data format.
+ filters : tensor
+ A 5-D Tensor with the same type as value and shape [height, width, output_channels, in_channels].
+ filter's in_channels dimension must match that of value.
+ output_shape : tensor
+ A 1-D Tensor representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1, 3 or 5.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NDHWC' and 'NCDHW' are supported.
+ dilations : list of ints
+ An int or list of ints that has length 1, 3 or 5, defaults to 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ pass
+
+
+class BatchNorm(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, *args, **kwargs):
+ pass
diff --git a/tensorlayer/backend/ops/tensorflow_backend.py b/tensorlayer/backend/ops/tensorflow_backend.py
new file mode 100644
index 0000000..74df53d
--- /dev/null
+++ b/tensorlayer/backend/ops/tensorflow_backend.py
@@ -0,0 +1,1021 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function
+from .tensorflow_nn import nchw_to_nhwc, nhwc_to_nchw
+import tensorflow as tf
+
+_dtypeDict = {
+ 'DType': tf.DType,
+ 'float16': tf.float16,
+ 'float32': tf.float32,
+ 'float64': tf.float64,
+ 'int8': tf.int8,
+ 'int16': tf.int16,
+ 'int32': tf.int32,
+ 'int64': tf.int64,
+ 'uint8': tf.uint8,
+ 'uint16': tf.uint16,
+ 'uint32': tf.uint32,
+ 'uint64': tf.uint64
+}
+
+DType = tf.DType
+float16 = tf.float16
+float32 = tf.float32
+float64 = tf.float64
+int8 = tf.int8
+int16 = tf.int16
+int32 = tf.int32
+int64 = tf.int64
+uint8 = tf.uint8
+uint16 = tf.uint16
+uint32 = tf.uint32
+uint64 = tf.uint64
+
+# isinstance input output
+# TensorLike = tf_ops._TensorLike
+
+
+def set_context(**kwargs):
+ raise Exception("Using TenosrFlow backend,You don't need to set context")
+
+
+def get_tensor_shape(x):
+ return x.get_shape().as_list()
+
+
+# initializers
+def zeros(shape, dtype=tf.float32):
+ """
+ Creates a tensor with all elements set to zero.
+
+ Parameters
+ ----------
+ shape : A list of integers
+ a tuple of integers, or a 1-D Tensor of type int32.
+ dtype : tensor
+ The DType of an element in the resulting Tensor
+
+ Returns
+ -------
+ A Tensor with all elements set to zero.
+
+ """
+ return tf.zeros(shape=shape, dtype=dtype)
+
+
+def ones(shape, dtype=tf.float32):
+ """
+ Creates a tensor with all elements set to ones.
+
+ Parameters
+ ----------
+ shape : A list of integers
+ a tuple of integers, or a 1-D Tensor of type int32.
+ dtype : tensor
+ The DType of an element in the resulting Tensor
+
+ Returns
+ -------
+ A Tensor with all elements set to zero.
+
+ """
+ return tf.ones(shape=shape, dtype=dtype)
+
+
+def constant(value, dtype=tf.float32, shape=None):
+ """
+ Creates a constant tensor from a tensor-like object.
+
+ Parameters
+ ----------
+ value : list
+ A constant value (or list) of output type dtype.
+ dtype : tensor
+ The type of the elements of the resulting tensor.
+ shape : tuple
+ Optional dimensions of resulting tensor.
+
+ Returns
+ -------
+ A Constant Tensor.
+
+ """
+ return tf.constant(value=value, dtype=dtype, shape=shape)
+
+
+def random_uniform(shape, minval=0, maxval=None, dtype=tf.float32, seed=None):
+ """
+ Outputs random values from a uniform distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ minval : int
+ The lower bound on the range of random values to generate (inclusive). Defaults to 0.
+ maxval : int
+ The upper bound on the range of random values to generate (exclusive). Defaults to 1 if dtype is floating point.
+ dtype : tensor
+ The type of the output: float16, float32, float64, int32, or int64.
+ seed : int
+ Used in combination with tf.random.set_seed to create a reproducible sequence of tensors across multiple calls.
+ Returns
+ -------
+ A tensor of the specified shape filled with random uniform values.
+
+ """
+ outputs = tf.random.uniform(shape=shape, minval=minval, maxval=maxval, dtype=dtype, seed=seed)
+ return outputs
+
+
+def random_normal(shape, mean=0.0, stddev=1.0, dtype=tf.dtypes.float32, seed=None):
+ """
+ Outputs random values from a normal distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ dtype : tensor
+ The type of the output.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns
+ -------
+ A tensor of the specified shape filled with random normal values.
+
+ """
+ outputs = tf.random.normal(shape=shape, mean=mean, stddev=stddev, dtype=dtype, seed=seed)
+ return outputs
+
+
+def truncated_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None):
+ """
+ Outputs random values from a truncated normal distribution.
+
+ Parameters
+ ----------
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ mean : float
+ The mean of the normal distribution
+ stddev : float
+ The standard deviation of the normal distribution.
+ dtype : tensor
+ The type of the output.
+ seed : A Python integer
+ Used to create a random seed for the distribution
+
+ Returns
+ -------
+ A tensor of the specified shape filled with random truncated normal values.
+
+ """
+ outputs = tf.random.truncated_normal(shape=shape, mean=mean, stddev=stddev, dtype=dtype, seed=seed)
+ return outputs
+
+
+def he_normal(shape, dtype, seed=None):
+ """
+ He normal initializer.
+
+ Parameters
+ ----------
+ seed : A Python integer.
+ Used to seed the random generator.
+ shape : tuple
+ A 1-D integer Tensor or Python array. The shape of the output tensor.
+ dtype : tensor
+ The type of the output.
+
+ Returns
+ -------
+ A tensor of the specified shape filled with he normal values.
+ """
+ return tf.initializers.he_normal(seed)(shape=shape, dtype=dtype)
+
+
+def Variable(initial_value, name, trainable=True):
+ """
+ Creates a new variable with value initial_value.
+
+ Parameters
+ ----------
+ initial_value : tensor
+ A Tensor, or Python object convertible to a Tensor
+ name : str
+ Optional name for the variable. Defaults to 'Variable' and gets uniquified automatically.
+ Returns
+ -------
+ Variable
+ """
+
+ var = tf.Variable(initial_value=initial_value, name=name, trainable=trainable)
+ return var
+
+
+class MatMul(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, a, b):
+ return tf.matmul(a, b)
+
+
+def matmul(a, b):
+ """
+ Multiplies matrix a by matrix b, producing a * b.
+
+ Parameters
+ ----------
+ a : tensor
+ type float16, float32, float64, int32, complex64, complex128 and rank > 1.
+ b : tensor
+ with same type and rank as a.
+
+ Returns
+ -------
+ A Tensor of the same type as a and b
+ """
+
+ outputs = tf.matmul(a, b)
+ return outputs
+
+
+def add(value, bias):
+ """
+ Returns x + y element-wise.
+
+ Parameters
+ ----------
+ value : tensor.
+ Must be one of the following types: bfloat16, half, float32, float64,
+ uint8, int8, int16, int32, int64, complex64, complex128, string.
+ bias : tensor
+ Must have the same type as a
+
+ Returns
+ -------
+ A Tensor. Has the same type as a.
+ """
+
+ outputs = tf.add(value, bias)
+ return outputs
+
+
+def dtypes(dt):
+ """
+ Data dtypes.
+
+ Parameters
+ ----------
+ dt : string
+ It could be 'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16',
+ 'int32', 'int64', 'float16', 'float32', 'float64', 'DType'.
+
+ Returns
+ -------
+ Data dtypes
+ """
+
+ if dt not in _dtypeDict.keys():
+ raise Exception("Unsupported dtype: {}".format(dt))
+ return _dtypeDict[dt]
+
+
+class Maximum(object):
+ def __init__(self):
+ pass
+
+ def __call__(self, x, y):
+ return tf.maximum(x=x, y=y)
+
+
+class Minimum(object):
+ def __init__(self):
+ pass
+
+ def __call__(self, x, y):
+ return tf.minimum(x=x, y=y)
+
+
+def minimum(x, y):
+ """
+ Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+
+ Parameters
+ ----------
+ x : tensor.
+ Must be one of the following types: bfloat16, half, float32, float64, int32, int64.
+ y : A Tensor.
+ Must have the same type as x.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x
+ """
+
+ outputs = tf.minimum(x=x, y=y)
+ return outputs
+
+
+class FlattenReshape(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, inputs):
+ dim = 1
+ for d in get_tensor_shape(inputs)[1:]:
+ dim *= d
+ return tf.reshape(inputs, [-1, dim])
+
+
+class Reshape(object):
+
+ def __init__(self, shape):
+ self.shape = shape
+
+ def __call__(self, tensor):
+ return tf.reshape(tensor, self.shape)
+
+
+def reshape(tensor, shape):
+ """
+ Reshapes a tensor.
+
+ Parameters
+ ----------
+ tensor : tensor
+ A Tensor.
+ shape : tensor
+ Defines the shape of the output tensor.
+ Returns
+ -------
+ A Tensor. Has the same type as tensor
+ """
+
+ return tf.reshape(tensor, shape)
+
+
+class Concat(object):
+
+ def __init__(self, axis):
+ super(Concat, self).__init__()
+ self.axis = axis
+
+ def __call__(self, values):
+ return tf.concat(values=values, axis=self.axis)
+
+
+def concat(values, axis):
+ """
+ Concatenates tensors along one dimension.
+
+ Parameters
+ ----------
+ values : list
+ A list of Tensor objects or a single Tensor
+ axis : int
+ 0-D int32 Tensor. Dimension along which to concatenate
+ Returns
+ -------
+ A Tensor resulting from concatenation of the input tensors.
+ """
+
+ return tf.concat(values, axis)
+
+
+def convert_to_tensor(value, dtype=None):
+ """
+ Converts the given value to a Tensor.
+
+ Parameters
+ ----------
+ value : object
+ An object whose type has a registered Tensor conversion function.
+ dtype : optional
+ Optional element type for the returned tensor. If missing, the type is inferred from the type of value.
+
+ Returns
+ -------
+ A Tensor based on value.
+ """
+
+ return tf.convert_to_tensor(value, dtype)
+
+
+def sqrt(x):
+ """
+ Computes square root of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x.
+ """
+ return tf.sqrt(x)
+
+
+class ReduceSum(object):
+
+ def __init__(self, axis=None):
+ self.axis = axis
+
+ def __call__(self, input):
+ return tf.reduce_sum(input, axis=self.axis)
+
+
+class ReduceMean(object):
+
+ def __init__(self, axis):
+ self.axis = axis
+
+ def __call__(self, inputs):
+ output = tf.reduce_mean(inputs, self.axis)
+ return output
+
+
+def reduce_mean(input_tensor, axis=None):
+ """
+ Computes the mean of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have numeric type.
+ axis : list
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ return tf.reduce_mean(input_tensor, axis=axis)
+
+
+class ReduceMax(object):
+
+ def __init__(self, axis):
+ self.axis = axis
+
+ def __call__(self, inputs):
+ output = tf.reduce_max(inputs, self.axis)
+ return output
+
+
+def reduce_max(input_tensor, axis=None):
+ """
+ Computes the maximum of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have real numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ return tf.reduce_max(input_tensor, axis=axis)
+
+
+def reduce_min(input_tensor, axis=None):
+ """
+ Computes the minimum of elements across dimensions of a tensor.
+
+ Parameters
+ ----------
+ input_tensor : tensor
+ The tensor to reduce. Should have real numeric type.
+ axis : int
+ The dimensions to reduce. If None (the default), reduces all dimensions.
+ Must be in the range [-rank(input_tensor), rank(input_tensor)).
+ name : str
+ A name for the operation (optional).
+
+ Returns
+ -------
+ The reduced tensor.
+ """
+
+ return tf.reduce_min(input_tensor, axis=axis)
+
+
+class Pad(object):
+ def __init__(self, paddings, mode="REFLECT"):
+ if mode not in ['CONSTANT', 'REFLECT', 'SYMMETRIC']:
+ raise Exception("Unsupported mode: {}".format(mode))
+ self.paddings = paddings
+ self.mode = mode
+
+ def __call__(self, x):
+ outputs = tf.pad(x, self.paddings, mode=self.mode, constant_values=0)
+ return outputs
+
+def pad(tensor, paddings, mode='CONSTANT', constant_values=0):
+ """
+ Pads a tensor.
+
+ Parameters
+ ----------
+ tensor : tensor
+ A Tensor.
+ paddings : tensor
+ A Tensor of type int32.
+ mode : str
+ One of "CONSTANT", "REFLECT", or "SYMMETRIC" (case-insensitive)
+ constant_values : int
+ In "CONSTANT" mode, the scalar pad value to use. Must be same type as tensor.
+
+ Returns
+ -------
+ A Tensor. Has the same type as tensor.
+ """
+
+ if mode not in ['CONSTANT', 'REFLECT', 'SYMMETRIC']:
+ raise Exception("Unsupported mode: {}".format(mode))
+ outputs = tf.pad(tensor, paddings, mode=mode, constant_values=constant_values)
+ return outputs
+
+
+class Unstack(object):
+
+ def __init__(self, axis, num=None):
+ self.axis = axis
+ self.num = num
+
+ def __call__(self, values):
+ return tf.unstack(values, num=self.num, axis=self.axis)
+
+
+class Stack(object):
+
+ def __init__(self, axis=0):
+ self.axis = axis
+
+ def __call__(self, values):
+ return tf.stack(values, axis=self.axis)
+
+
+def stack(values, axis=0):
+ """
+ Stacks a list of rank-R tensors into one rank-(R+1) tensor.
+
+ Parameters
+ ----------
+ values : list
+ A list of Tensor objects with the same shape and type.
+ axis : int
+ An int. The axis to stack along. Defaults to the first dimension.
+ Negative values wrap around, so the valid range is [-(R+1), R+1).
+
+ Returns
+ -------
+ A stacked Tensor with the same type as values.
+ """
+
+ return tf.stack(values, axis=axis)
+
+
+class Meshgrid(object):
+ def __init__(self, indexing='xy'):
+ super(Meshgrid, self).__init__()
+ self.index = indexing
+
+ def __call__(self, inputs):
+ return tf.meshgrid(inputs)
+
+
+def meshgrid(*args, **kwargs):
+ """
+ Broadcasts parameters for evaluation on an N-D grid.
+
+ Parameters
+ ----------
+ x : tensor
+ Tensors with rank 1.
+ y : tensor
+ Tensors with rank 1.
+
+ Returns
+ -------
+ A list of N Tensors with rank N.
+ """
+
+ return tf.meshgrid(*args, **kwargs)
+
+
+def range(start, limit=None, delta=1, dtype=None):
+ """
+ Creates a sequence of numbers.
+
+ Parameters
+ ----------
+ start : tensor
+ A 0-D Tensor (scalar). Acts as first entry in the range if limit is not None;
+ otherwise, acts as range limit and first entry defaults to 0.
+ limit : tensor
+ A 0-D Tensor (scalar). Upper limit of sequence, exclusive. If None,
+ defaults to the value of start while the first entry of the range defaults to 0.
+ delta : tensor
+ A 0-D Tensor (scalar). Number that increments start. Defaults to 1.
+ dtype : type
+ The type of the elements of the resulting tensor.
+
+ Returns
+ -------
+ An 1-D Tensor of type dtype.
+ """
+
+ if limit is None:
+ outputs = tf.range(start, delta=delta, dtype=dtype)
+ else:
+ outputs = tf.range(start, limit, delta=delta, dtype=dtype)
+ return outputs
+
+
+class ExpandDims(object):
+
+ def __init__(self, axis):
+ self.axis = axis
+
+ def __call__(self, input):
+ return tf.expand_dims(input, axis=self.axis)
+
+
+def expand_dims(input, axis):
+ """
+ Inserts a dimension of 1 into a tensor's shape.
+
+ Parameters
+ ----------
+ input : tensor
+ A Tensor.
+ axis : int
+ 0-D (scalar). Specifies the dimension index at which to expand the shape of input.
+ Must be in the range [-rank(input) - 1, rank(input)].
+
+ Returns
+ -------
+ A Tensor with the same data as input, but its shape has an additional dimension of size 1 added.
+ """
+
+ return tf.expand_dims(input, axis)
+
+
+class Tile(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, input, multiples):
+ return tf.tile(input, multiples)
+
+
+def tile(input, multiples):
+ """
+ Constructs a tensor by tiling a given tensor.
+
+ Parameters
+ ----------
+ input : tensor
+ A Tensor. 1-D or higher.
+ multiples : tensor
+ Must be one of the following types: int32, int64. 1-D.
+ Length must be the same as the number of dimensions in input
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ return tf.tile(input, multiples)
+
+
+class Cast(object):
+
+ def __init__(self, dtype):
+ self.dtype = dtype
+
+ def __call__(self, x):
+ return tf.cast(x, dtype=self.dtype)
+
+
+def cast(x, dtype):
+ """
+ Casts a tensor to a new type.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor or SparseTensor or IndexedSlices of numeric type.
+ It could be uint8, uint16, uint32, uint64, int8, int16, int32, int64, float16, float32, float64.
+ dtype : dtpye
+ The destination type. The list of supported dtypes is the same as x
+
+ Returns
+ -------
+ A Tensor or SparseTensor or IndexedSlices with same shape as x and same type as dtype.
+ """
+
+ return tf.cast(x, dtype=dtype)
+
+
+class Transpose(object):
+
+ def __init__(self, perm, conjugate=False):
+ self.perm = perm
+ self.conjugate = conjugate
+
+ def __call__(self, a):
+ return tf.transpose(a, self.perm, self.conjugate)
+
+
+def transpose(a, perm=None, conjugate=False):
+ """
+ Transposes a.
+
+ Parameters
+ ----------
+ a : tensor
+ A Tensor.
+ perm : list / int
+ A permutation of the dimensions of a.
+ conjugate : bool
+ Setting it to True is mathematically equivalent to tf.math.conj(tf.transpose(input)).
+
+ Returns
+ -------
+ A transposed Tensor.
+ """
+
+ return tf.transpose(a, perm, conjugate)
+
+
+def gather_nd(params, indices, batch_dims=0):
+ """
+ Gather slices from params into a Tensor with shape specified by indices.
+
+ Parameters
+ ----------
+ params : tensor
+ The tensor from which to gather values.
+ indices : tensor
+ Must be one of the following types: int32, int64. Index tensor.
+ batch_dims : int
+ An integer or a scalar 'Tensor'. The number of batch dimensions.
+
+ Returns
+ -------
+ A Tensor. Has the same type as params.
+ """
+
+ return tf.gather_nd(params, indices, batch_dims)
+
+
+def clip_by_value(t, clip_value_min, clip_value_max):
+ """
+ Clips tensor values to a specified min and max.
+
+ Parameters
+ ----------
+ t : tensor
+ A Tensor or IndexedSlices
+ clip_value_min : tensor
+ A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by
+ clip_value_max : tensor
+ A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by
+
+ Returns
+ -------
+ A clipped Tensor or IndexedSlices.
+ """
+
+ return tf.clip_by_value(t, clip_value_min, clip_value_max)
+
+
+def split(value, num_or_size_splits, axis=0, num=None):
+ """
+ Splits a tensor into sub tensors.
+
+ Parameters
+ ----------
+ value : tensor
+ The Tensor to split.
+ num_or_size_splits : list
+ Either an integer indicating the number of splits along split_dim or a 1-D integer Tensor or
+ Python list containing the sizes of each output tensor along split_dim.
+ axis : int
+ The dimension along which to split. Must be in the range [-rank(value), rank(value)). Defaults to 0.
+ num : int
+ used to specify the number of outputs when it cannot be inferred from the shape of size_splits.
+
+ Returns
+ -------
+ Tensor objects resulting from splitting value.
+ """
+
+ return tf.split(value=value, num_or_size_splits=num_or_size_splits, axis=axis, num=num)
+
+
+def floor(x):
+ return tf.floor(x)
+
+
+def gather(params, indices):
+ return tf.gather(params, indices)
+
+
+def linspace(start, stop, num):
+ return tf.linspace(start, stop, num)
+
+
+def slice(inputs, starts, sizes):
+ return tf.slice(inputs, starts, sizes)
+
+
+def add_n(inputs):
+ return tf.add_n(inputs)
+
+
+class OneHot(object):
+
+ def __init__(self, depth, on_value, off_value, axis, dtype):
+ self.depth = depth
+ self.on_value = on_value
+ self.off_value = off_value
+ self.axis = axis
+ self.dtype = dtype
+
+ def __call__(self, inputs, *args, **kwargs):
+ outputs = tf.one_hot(
+ inputs, self.depth, on_value=self.on_value, off_value=self.off_value, axis=self.axis, dtype=self.dtype
+ )
+ return outputs
+
+
+class L2Normalize(object):
+
+ def __init__(self, axis=None, epsilon=1e-12):
+ self.axis = axis
+ self.epsilon = epsilon
+
+ def __call__(self, input, *args, **kwargs):
+ outputs = tf.math.l2_normalize(input, axis=self.axis, epsilon=self.epsilon)
+ return outputs
+
+
+class EmbeddingLookup(object):
+
+ def __init__(self, max_norm=None):
+ self.max_norm = max_norm
+
+ def __call__(self, params, ids, *args, **kwargs):
+ outputs = tf.nn.embedding_lookup(params=params, ids=ids, max_norm=self.max_norm)
+ return outputs
+
+
+class NCELoss(object):
+
+ def __init__(self, num_true=1, sampled_values=None, remove_accidental_hits=False):
+ self.num_true = num_true
+ self.sampled_values = sampled_values
+ self.remove_accidental_hits = remove_accidental_hits
+
+ def __call__(self, weights, biases, labels, inputs, num_sampled, num_classes):
+ outputs = tf.nn.nce_loss(
+ weights=weights, biases=biases, inputs=inputs, labels=labels, num_sampled=num_sampled,
+ num_classes=num_classes
+ )
+ return outputs
+
+
+class Not_equal(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x, y):
+ return tf.not_equal(x, y)
+
+
+class Count_nonzero(object):
+
+ def __init__(self, keepdims=None, dtype=int64):
+ self.keepdims = keepdims
+ self.dtype = dtype
+
+ def __call__(self, input, axis=None):
+ return tf.math.count_nonzero(input, axis=axis, keepdims=self.keepdims, dtype=self.dtype)
+
+
+
+class Resize:
+
+ def __init__(self, scale, method, antialias=False, data_format='channels_last', ksize=None):
+ self.method = method
+ self.antialias = antialias
+ self.scale = scale
+ self.data_format = data_format
+
+ def __call__(self, inputs):
+ if self.data_format == 'channels_first':
+ inputs = nchw_to_nhwc(inputs)
+ if len(get_tensor_shape(inputs)) == 4:
+ output_size = [int(inputs.shape[1] * self.scale[0]), int(inputs.shape[2] * self.scale[1])]
+ else:
+ raise ("The inputs shape must be 4-D Tensor.")
+ outputs = tf.image.resize(inputs, size=output_size, method=self.method, antialias=self.antialias)
+ if self.data_format == 'channels_first':
+ outputs = nhwc_to_nchw(outputs)
+ return outputs
+
+
+def resize(inputs, output_size, method, antialias):
+ return tf.image.resize(inputs, size=output_size, method=method, antialias=antialias)
+
+
+class ZeroPadding1D(object):
+
+ def __init__(self, padding):
+ self.zeropad = tf.keras.layers.ZeroPadding1D(padding=padding)
+
+ def __call__(self, inputs):
+ return self.zeropad(inputs)
+
+
+class ZeroPadding2D(object):
+
+ def __init__(self, padding):
+ self.zeropad = tf.keras.layers.ZeroPadding2D(padding=padding)
+
+ def __call__(self, inputs):
+ return self.zeropad(inputs)
+
+
+class ZeroPadding3D(object):
+
+ def __init__(self, padding):
+ self.zeropad = tf.keras.layers.ZeroPadding3D(padding=padding)
+
+ def __call__(self, inputs):
+ return self.zeropad(inputs)
+
+
+class Sign(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return tf.sign(x)
+
+def ceil(x):
+ return tf.math.ceil(x)
+
+def multiply(x, y):
+ return tf.multiply(x, y)
+
+def divide(x, y):
+ return tf.divide(x, y)
+
+def identity(x):
+ return tf.identity(x)
+
+class BatchToSpace(object):
+ def __init__(self, block_size, crops):
+ self.bolock_size = block_size
+ self.crops = crops
+
+ def __call__(self, input_x):
+ return tf.batch_to_space(input=input_x, block_shape=self.bolock_size, crops=self.crops)
+
+class DepthToSpace(object):
+ def __init__(self, block_size, data_format='NHWC'):
+ self.block_size = block_size
+ self.data_format = data_format
+
+ def __call__(self, input):
+ return tf.nn.depth_to_space(input, block_size=self.block_size, data_format=self.data_format)
diff --git a/tensorlayer/backend/ops/tensorflow_nn.py b/tensorlayer/backend/ops/tensorflow_nn.py
new file mode 100644
index 0000000..71d978f
--- /dev/null
+++ b/tensorlayer/backend/ops/tensorflow_nn.py
@@ -0,0 +1,1519 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import tensorflow as tf
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.training import moving_averages
+
+# loss function
+sparse_softmax_cross_entropy_with_logits = tf.nn.sparse_softmax_cross_entropy_with_logits
+sigmoid_cross_entropy_with_logits = tf.nn.sigmoid_cross_entropy_with_logits
+
+
+def padding_format(padding):
+ """
+ Checks that the padding format correspond format.
+
+ Parameters
+ ----------
+ padding : str
+ Must be one of the following:"same", "SAME", "VALID", "valid"
+
+ Returns
+ -------
+ str "SAME" or "VALID"
+ """
+
+ if padding in ["SAME", "same"]:
+ padding = "SAME"
+ elif padding in ["VALID", "valid"]:
+ padding = "VALID"
+ elif padding == None:
+ padding = None
+ else:
+ raise Exception("Unsupported padding: " + str(padding))
+ return padding
+
+
+def preprocess_1d_format(data_format, padding):
+ """
+ Checks that the 1-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NWC","NCW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NWC" or "NCW" and "SAME" or "VALID"
+ """
+ if data_format in ["channels_last", "NWC"]:
+ data_format = "NWC"
+ elif data_format in ["channels_first", "NCW"]:
+ data_format = "NCW"
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def preprocess_2d_format(data_format, padding):
+ """
+ Checks that the 2-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NHWC","NCHW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NHWC" or "NCHW" and "SAME" or "VALID"
+ """
+
+ if data_format in ["channels_last", "NHWC"]:
+ data_format = "NHWC"
+ elif data_format in ["channels_first", "NCHW"]:
+ data_format = "NCHW"
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def preprocess_3d_format(data_format, padding):
+ """
+ Checks that the 3-D dataformat format correspond format.
+
+ Parameters
+ ----------
+ data_format : str
+ Must be one of the following:"channels_last","NDHWC","NCDHW","channels_first"
+ padding : str
+ Must be one of the following:"same","valid","SAME","VALID"
+
+ Returns
+ -------
+ str "NDHWC" or "NCDHW" and "SAME" or "VALID"
+ """
+
+ if data_format in ['channels_last', 'NDHWC']:
+ data_format = 'NDHWC'
+ elif data_format in ['channels_first', 'NCDHW']:
+ data_format = 'NCDHW'
+ elif data_format == None:
+ data_format = None
+ else:
+ raise Exception("Unsupported data format: " + str(data_format))
+ padding = padding_format(padding)
+ return data_format, padding
+
+
+def nchw_to_nhwc(x):
+ """
+ Channels first to channels last
+
+ Parameters
+ ----------
+ x : tensor
+ channels first tensor data
+
+ Returns
+ -------
+ channels last tensor data
+ """
+
+ if len(x.shape) == 3:
+ x = tf.transpose(x, (0, 2, 1))
+ elif len(x.shape) == 4:
+ x = tf.transpose(x, (0, 2, 3, 1))
+ elif len(x.shape) == 5:
+ x = tf.transpose(x, (0, 2, 3, 4, 1))
+ else:
+ raise Exception("Unsupported dimensions")
+ return x
+
+
+def nhwc_to_nchw(x):
+ """
+ Channles last to channels first
+
+ Parameters
+ ----------
+ x : tensor
+ channels last tensor data
+
+ Returns
+ -------
+ channels first tensor data
+ """
+
+ if len(x.shape) == 3:
+ x = tf.transpose(x, (0, 2, 1))
+ elif len(x.shape) == 4:
+ x = tf.transpose(x, (0, 3, 1, 2))
+ elif len(x.shape) == 5:
+ x = tf.transpose(x, (0, 4, 1, 2, 3))
+ else:
+ raise Exception("Unsupported dimensions")
+ return x
+
+
+class ReLU(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return tf.nn.relu(x)
+
+
+def relu(x):
+ """
+ Computes rectified linear: max(features, 0).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: float32, float64, int32, uint8, int16,
+ int8, int64, bfloat16, uint16, half, uint32, uint64, qint8.
+
+ Returns
+ -------
+ A Tensor. Has the same type as features.
+ """
+
+ return tf.nn.relu(x)
+
+
+class ReLU6(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return tf.nn.relu6(x)
+
+
+def relu6(x):
+ """
+ Computes Rectified Linear 6: min(max(features, 0), 6).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: float32, float64, int32, uint8, int16,
+ int8, int64, bfloat16, uint16, half, uint32, uint64, qint8.
+
+ Returns
+ -------
+ A Tensor with the same type as features.
+ """
+
+ return tf.nn.relu6(x)
+
+
+class LeakyReLU(object):
+
+ def __init__(self, alpha=0.2):
+ self.alpha = alpha
+
+ def __call__(self, x):
+ return tf.nn.leaky_relu(x, alpha=self.alpha)
+
+
+def leaky_relu(x, alpha=0.2):
+ """
+ Compute the Leaky ReLU activation function.
+
+ Parameters
+ ----------
+ x : tensor
+ representing preactivation values. Must be one of the following types:
+ float16, float32, float64, int32, int64.
+
+ Returns
+ -------
+ The activation value.
+ """
+
+ return tf.nn.leaky_relu(x, alpha=alpha)
+
+
+class Softplus(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return tf.nn.softplus(x)
+
+
+def softplus(x):
+ """
+ Computes softplus: log(exp(features) + 1).
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: half, bfloat16, float32, float64.
+
+ Returns
+ -------
+ A Tensor. Has the same type as features.
+ """
+
+ return tf.nn.softplus(x)
+
+
+class Tanh(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return tf.nn.tanh(x)
+
+
+def tanh(x):
+ """
+ Computes hyperbolic tangent of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128.
+
+ Returns
+ -------
+ A Tensor. Has the same type as x.
+ """
+
+ return tf.nn.tanh(x)
+
+
+class Sigmoid(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return tf.nn.sigmoid(x)
+
+
+def sigmoid(x):
+ """
+ Computes sigmoid of x element-wise.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float16, float32, float64, complex64, or complex128.
+
+ Returns
+ -------
+ A Tensor with the same type as x.
+ """
+
+ return tf.nn.sigmoid(x)
+
+
+class Softmax(object):
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return tf.nn.softmax(x)
+
+
+def softmax(logits, axis=None):
+ """
+ Computes softmax activations.
+
+ Parameters
+ ----------
+ logits : tensor
+ Must be one of the following types: half, float32, float64.
+ axis : int
+ The dimension softmax would be performed on. The default is -1 which indicates the last dimension.
+
+ Returns
+ -------
+ A Tensor. Has the same type and shape as logits.
+ """
+
+ return tf.nn.softmax(logits, axis)
+
+
+class Dropout(object):
+
+ def __init__(self, keep, seed=0):
+ self.keep = keep
+ self.seed = seed
+
+ def __call__(self, inputs, *args, **kwargs):
+ outputs = tf.nn.dropout(inputs, rate=1 - (self.keep), seed=self.seed)
+ return outputs
+
+
+class BiasAdd(object):
+ """
+ Adds bias to value.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float, double, int64, int32, uint8, int16, int8, complex64, or complex128.
+ bias : tensor
+ Must be the same type as value unless value is a quantized type,
+ in which case a different quantized type may be used.
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ def __init__(self, data_format=None):
+ self.data_format = data_format
+
+ def __call__(self, x, bias):
+ return tf.nn.bias_add(x, bias, data_format=self.data_format)
+
+
+def bias_add(x, bias, data_format=None, name=None):
+ """
+ Adds bias to value.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor with type float, double, int64, int32, uint8, int16, int8, complex64, or complex128.
+ bias : tensor
+ Must be the same type as value unless value is a quantized type,
+ in which case a different quantized type may be used.
+ data_format : A string.
+ 'N...C' and 'NC...' are supported.
+ name : str
+ A name for the operation (optional).
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ x = tf.nn.bias_add(x, bias, data_format=data_format, name=name)
+ return x
+
+
+class Conv1D(object):
+
+ def __init__(self, stride, padding, data_format='NWC', dilations=None, out_channel=None, k_size=None):
+ self.stride = stride
+ self.dilations = dilations
+ self.data_format, self.padding = preprocess_1d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ outputs = tf.nn.conv1d(
+ input=input,
+ filters=filters,
+ stride=self.stride,
+ padding=self.padding,
+ data_format=self.data_format,
+ dilations=self.dilations,
+ # name=name
+ )
+ return outputs
+
+
+def conv1d(input, filters, stride, padding, data_format='NWC', dilations=None):
+ """
+ Computes a 1-D convolution given 3-D input and filter tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ A 3D Tensor. Must be of type float16, float32, or float64
+ filters : tensor
+ A 3D Tensor. Must have the same type as input.
+ stride : int of list
+ An int or list of ints that has length 1 or 3. The number of entries by which the filter is moved right at each step.
+ padding : string
+ 'SAME' or 'VALID'
+ data_format : string
+ An optional string from "NWC", "NCW". Defaults to "NWC", the data is stored in the order of
+ [batch, in_width, in_channels]. The "NCW" format stores data as [batch, in_channels, in_width].
+ dilations : int or list
+ An int or list of ints that has length 1 or 3 which defaults to 1.
+ The dilation factor for each dimension of input. If set to k > 1,
+ there will be k-1 skipped cells between each filter element on that dimension.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ A name for the operation (optional).
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ data_format, padding = preprocess_1d_format(data_format, padding)
+ outputs = tf.nn.conv1d(
+ input=input,
+ filters=filters,
+ stride=stride,
+ padding=padding,
+ data_format=data_format,
+ dilations=dilations,
+ # name=name
+ )
+ return outputs
+
+
+class Conv2D(object):
+
+ def __init__(self, strides, padding, data_format='NHWC', dilations=None, out_channel=None, k_size=None):
+ self.strides = strides
+ self.dilations = dilations
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ outputs = tf.nn.conv2d(
+ input=input,
+ filters=filters,
+ strides=self.strides,
+ padding=self.padding,
+ data_format=self.data_format,
+ dilations=self.dilations,
+ )
+ return outputs
+
+
+def conv2d(input, filters, strides, padding, data_format='NHWC', dilations=None):
+ """
+ Computes a 2-D convolution given 4-D input and filters tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ Must be one of the following types: half, bfloat16, float32, float64. A 4-D tensor.
+ The dimension order is interpreted according to the value of data_format, see below for details.
+ filters : tensor
+ Must have the same type as input. A 4-D tensor of shape [filter_height, filter_width, in_channels, out_channels]
+ strides : int of list
+ The stride of the sliding window for each dimension of input. If a single value is given it is replicated in the H and W dimension.
+ By default the N and C dimensions are set to 1. The dimension order is determined by the value of data_format, see below for details.
+ padding : string
+ "SAME" or "VALID"
+ data_format : string
+ "NHWC", "NCHW". Defaults to "NHWC".
+ dilations : list or ints
+ list of ints that has length 1, 2 or 4, defaults to 1. The dilation factor for each dimension ofinput.
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ data_format, padding = preprocess_2d_format(data_format, padding)
+ outputs = tf.nn.conv2d(
+ input=input,
+ filters=filters,
+ strides=strides,
+ padding=padding,
+ data_format=data_format,
+ dilations=dilations,
+ )
+ return outputs
+
+
+class Conv3D(object):
+
+ def __init__(self, strides, padding, data_format='NDHWC', dilations=None, out_channel=None, k_size=None):
+ self.strides = strides
+ self.dilations = dilations
+ self.data_format, self.padding = preprocess_3d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ outputs = tf.nn.conv3d(
+ input=input,
+ filters=filters,
+ strides=self.strides,
+ padding=self.padding,
+ data_format=self.data_format,
+ dilations=self.dilations,
+ )
+ return outputs
+
+
+def conv3d(input, filters, strides, padding, data_format='NDHWC', dilations=None):
+ """
+ Computes a 3-D convolution given 5-D input and filters tensors.
+
+ Parameters
+ ----------
+ input : tensor
+ Must be one of the following types: half, bfloat16, float32, float64.
+ Shape [batch, in_depth, in_height, in_width, in_channels].
+ filters : tensor
+ Must have the same type as input. Shape [filter_depth, filter_height, filter_width, in_channels, out_channels].
+ in_channels must match between input and filters.
+ strides : list of ints
+ A list of ints that has length >= 5. 1-D tensor of length 5.
+ The stride of the sliding window for each dimension of input.
+ Must have strides[0] = strides[4] = 1.
+ padding : string
+ A string from: "SAME", "VALID". The type of padding algorithm to use.
+ data_format : string
+ An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC". The data format of the input and output data.
+ With the default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels].
+ Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width].
+ dilations : list of ints
+ Defaults to [1, 1, 1, 1, 1]. 1-D tensor of length 5. The dilation factor for each dimension of input.
+ If set to k > 1, there will be k-1 skipped cells between each filter element on that dimension.
+ The dimension order is determined by the value of data_format, see above for details.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ data_format, padding = preprocess_3d_format(data_format, padding)
+ outputs = tf.nn.conv3d(
+ input=input,
+ filters=filters,
+ strides=strides,
+ padding=padding,
+ data_format=data_format, # 'NDHWC',
+ dilations=dilations, # [1, 1, 1, 1, 1],
+ # name=name,
+ )
+ return outputs
+
+
+def lrn(inputs, depth_radius, bias, alpha, beta):
+ """
+ Local Response Normalization.
+
+ Parameters
+ ----------
+ inputs : tensor
+ Must be one of the following types: half, bfloat16, float32. 4-D.
+ depth_radius : int
+ Defaults to 5. 0-D. Half-width of the 1-D normalization window.
+ bias : float
+ Defaults to 1. An offset (usually positive to avoid dividing by 0).
+ alpha : float
+ Defaults to 1. A scale factor, usually positive.
+ beta : float
+ Defaults to 0.5. An exponent.
+
+ Returns
+ -------
+ A Tensor. Has the same type as input.
+ """
+
+ outputs = tf.nn.lrn(inputs, depth_radius=depth_radius, bias=bias, alpha=alpha, beta=beta)
+ return outputs
+
+
+def moments(x, axes, shift=None, keepdims=False):
+ """
+ Calculates the mean and variance of x.
+
+ Parameters
+ ----------
+ x : tensor
+ A Tensor
+ axes : list or ints
+ Axes along which to compute mean and variance.
+ shift : int
+ Not used in the current implementation.
+ keepdims : bool
+ produce moments with the same dimensionality as the input.
+
+ Returns
+ -------
+ Two Tensor objects: mean and variance.
+ """
+
+ outputs = tf.nn.moments(x, axes, shift, keepdims)
+ return outputs
+
+
+class MaxPool(object):
+
+ def __init__(self, ksize, strides, padding, data_format=None):
+ self.ksize = ksize
+ self.strides = strides
+ self.data_format = data_format
+ self.padding = padding
+
+ def __call__(self, inputs):
+ if inputs.ndim == 3:
+ self.data_format, self.padding = preprocess_1d_format(data_format=self.data_format, padding=self.padding)
+ elif inputs.ndim == 4:
+ self.data_format, self.padding = preprocess_2d_format(data_format=self.data_format, padding=self.padding)
+ elif inputs.ndim == 5:
+ self.data_format, self.padding = preprocess_3d_format(data_format=self.data_format, padding=self.padding)
+
+ outputs = tf.nn.max_pool(
+ input=inputs, ksize=self.ksize, strides=self.strides, padding=self.padding, data_format=self.data_format
+ )
+ return outputs
+
+
+def max_pool(input, ksize, strides, padding, data_format=None):
+ """
+ Performs the max pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels] if data_format does not start
+ with "NC" (default), or [batch_size, num_channels] + input_spatial_shape if data_format starts with "NC".
+ Pooling happens over the spatial dimensions only.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The max pooled output tensor.
+ """
+
+ if input.ndim == 3:
+ data_format, padding = preprocess_1d_format(data_format=data_format, padding=padding)
+ elif input.ndim == 4:
+ data_format, padding = preprocess_2d_format(data_format=data_format, padding=padding)
+ elif input.ndim == 5:
+ data_format, padding = preprocess_3d_format(data_format=data_format, padding=padding)
+
+ outputs = tf.nn.max_pool(input=input, ksize=ksize, strides=strides, padding=padding, data_format=data_format)
+ return outputs
+
+
+class AvgPool(object):
+
+ def __init__(self, ksize, strides, padding, data_format=None):
+ self.ksize = ksize
+ self.strides = strides
+ self.data_format = data_format
+ self.padding = padding_format(padding)
+
+ def __call__(self, inputs):
+ outputs = tf.nn.avg_pool(
+ input=inputs, ksize=self.ksize, strides=self.strides, padding=self.padding, data_format=self.data_format
+ )
+ return outputs
+
+
+def avg_pool(input, ksize, strides, padding):
+ """
+ Performs the avg pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels]
+ if data_format does not start with "NC" (default), or [batch_size, num_channels] + input_spatial_shape
+ if data_format starts with "NC". Pooling happens over the spatial dimensions only.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, N or N+2.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ name : string
+ Optional name for the operation.
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The average pooled output tensor.
+ """
+
+ padding = padding_format(padding)
+ outputs = tf.nn.avg_pool(
+ input=input,
+ ksize=ksize,
+ strides=strides,
+ padding=padding,
+ )
+ return outputs
+
+
+def max_pool3d(input, ksize, strides, padding, data_format=None):
+ """
+ Performs the max pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of the format specified by data_format.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ "NDHWC", "NCDHW". Defaults to "NDHWC". The data format of the input and output data.
+ With the default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels].
+ Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width].
+ name : string
+ A name for the operation (optional).
+
+ Returns
+ -------
+ A Tensor of format specified by data_format. The max pooled output tensor.
+ """
+
+ data_format, padding = preprocess_3d_format(data_format, padding)
+ outputs = tf.nn.max_pool3d(
+ input=input,
+ ksize=ksize,
+ strides=strides,
+ padding=padding,
+ data_format=data_format,
+ )
+ return outputs
+
+
+def avg_pool3d(input, ksize, strides, padding, data_format=None):
+ """
+ Performs the average pooling on the input.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of shape [batch, height, width, channels] and type float32, float64, qint8, quint8, or qint32.
+ ksize : int or list of ints
+ An int or list of ints that has length 1, 3 or 5. The size of the window for each dimension of the input tensor.
+ strides : int or list of ints
+ An int or list of ints that has length 1, 3 or 5.
+ The stride of the sliding window for each dimension of the input tensor.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NDHWC' and 'NCDHW' are supported.
+ name : string
+ Optional name for the operation.
+
+ Returns
+ -------
+ A Tensor with the same type as value. The average pooled output tensor.
+ """
+
+ data_format, padding = preprocess_3d_format(data_format, padding)
+ outputs = tf.nn.avg_pool3d(
+ input=input,
+ ksize=ksize,
+ strides=strides,
+ padding=padding,
+ data_format=data_format,
+ )
+ return outputs
+
+
+def pool(input, window_shape, pooling_type, strides=None, padding='VALID', data_format=None, dilations=None, name=None):
+ """
+ Performs an N-D pooling operation.
+
+ Parameters
+ ----------
+ input : tensor
+ Tensor of rank N+2, of shape [batch_size] + input_spatial_shape + [num_channels]
+ if data_format does not start with "NC" (default), or [batch_size, num_channels] + input_spatial_shape
+ if data_format starts with "NC". Pooling happens over the spatial dimensions only.
+ window_shape : int
+ Sequence of N ints >= 1.
+ pooling_type : string
+ Specifies pooling operation, must be "AVG" or "MAX".
+ strides : ints
+ Sequence of N ints >= 1. Defaults to [1]*N. If any value of strides is > 1, then all values of dilation_rate must be 1.
+ padding : string
+ The padding algorithm, must be "SAME" or "VALID". Defaults to "SAME".
+ See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ Specifies whether the channel dimension of the input and output is the last dimension (default, or if data_format does not start with "NC"),
+ or the second dimension (if data_format starts with "NC").
+ For N=1, the valid values are "NWC" (default) and "NCW". For N=2, the valid values are "NHWC" (default) and "NCHW".
+ For N=3, the valid values are "NDHWC" (default) and "NCDHW".
+ dilations : list of ints
+ Dilation rate. List of N ints >= 1. Defaults to [1]*N. If any value of dilation_rate is > 1, then all values of strides must be 1.
+ name : string
+ Optional. Name of the op.
+
+ Returns
+ -------
+ Tensor of rank N+2, of shape [batch_size] + output_spatial_shape + [num_channels]
+ """
+ if pooling_type in ["MAX", "max"]:
+ pooling_type = "MAX"
+ elif pooling_type in ["AVG", "avg"]:
+ pooling_type = "AVG"
+ else:
+ raise ValueError('Unsupported pool_mode: ' + str(pooling_type))
+ padding = padding_format(padding)
+ outputs = tf.nn.pool(
+ input=input,
+ window_shape=window_shape,
+ pooling_type=pooling_type,
+ strides=strides,
+ padding=padding,
+ data_format=data_format,
+ dilations=dilations,
+ name=name,
+ )
+ return outputs
+
+
+class DepthwiseConv2d(object):
+
+ def __init__(self, strides, padding, data_format=None, dilations=None, ksize=None, channel_multiplier=1):
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+ self.strides = strides
+ self.dilations = dilations
+
+ def __call__(self, input, filter):
+ outputs = tf.nn.depthwise_conv2d(
+ input=input,
+ filter=filter,
+ strides=self.strides,
+ padding=self.padding,
+ data_format=self.data_format,
+ dilations=self.dilations,
+ )
+ return outputs
+
+
+def depthwise_conv2d(input, filter, strides, padding, data_format=None, dilations=None, name=None):
+ """
+ Depthwise 2-D convolution.
+
+ Parameters
+ ----------
+ input : tensor
+ 4-D with shape according to data_format.
+ filter : tensor
+ 4-D with shape [filter_height, filter_width, in_channels, channel_multiplier].
+ strides : list
+ 1-D of size 4. The stride of the sliding window for each dimension of input.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ The data format for input. Either "NHWC" (default) or "NCHW".
+ dilations : list
+ 1-D of size 2. The dilation rate in which we sample input values across the height and width dimensions in atrous convolution.
+ If it is greater than 1, then all values of strides must be 1.
+ name : string
+ A name for this operation (optional).
+
+ Returns
+ -------
+ A 4-D Tensor with shape according to data_format.
+ E.g., for "NHWC" format, shape is [batch, out_height, out_width, in_channels * channel_multiplier].
+ """
+
+ data_format, padding = preprocess_2d_format(data_format, padding)
+ outputs = tf.nn.depthwise_conv2d(
+ input=input,
+ filter=filter,
+ strides=strides,
+ padding=padding,
+ data_format=data_format,
+ dilations=dilations,
+ name=name,
+ )
+ return outputs
+
+
+class Conv1d_transpose(object):
+
+ def __init__(
+ self, strides, padding, data_format='NWC', dilations=None, out_channel=None, k_size=None, in_channels=None
+ ):
+ self.strides = strides
+ self.dilations = dilations
+ self.data_format, self.padding = preprocess_1d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ batch_size = input.shape[0]
+ if self.data_format == 'NWC':
+ w_axis, c_axis = 1, 2
+ else:
+ w_axis, c_axis = 2, 1
+
+ input_shape = input.shape.as_list()
+ filters_shape = filters.shape.as_list()
+ input_w = input_shape[w_axis]
+ filters_w = filters_shape[0]
+ output_channels = filters_shape[1]
+ dilations_w = 1
+
+ if isinstance(self.strides, int):
+ strides_w = self.strides
+ else:
+ strides_list = list(self.strides)
+ strides_w = strides_list[w_axis]
+
+ if self.dilations is not None:
+ if isinstance(self.dilations, int):
+ dilations_w = self.dilations
+ else:
+ dilations_list = list(self.dilations)
+ dilations_w = dilations_list[w_axis]
+
+ filters_w = filters_w + (filters_w - 1) * (dilations_w - 1)
+ assert self.padding in {'SAME', 'VALID'}
+ if self.padding == 'VALID':
+ output_w = input_w * strides_w + max(filters_w - strides_w, 0)
+ elif self.padding == 'SAME':
+ output_w = input_w * strides_w
+
+ if self.data_format == 'NCW':
+ output_shape = (batch_size, output_channels, output_w)
+ else:
+ output_shape = (batch_size, output_w, output_channels)
+ output_shape = tf.stack(output_shape)
+ outputs = tf.nn.conv1d_transpose(
+ input=input,
+ filters=filters,
+ output_shape=output_shape,
+ strides=self.strides,
+ padding=self.padding,
+ data_format=self.data_format,
+ dilations=self.dilations,
+ )
+ return outputs
+
+
+def conv1d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NWC', dilations=None, name=None
+):
+ """
+ The transpose of conv1d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 3-D Tensor of type float and shape [batch, in_width, in_channels]
+ for NWC data format or [batch, in_channels, in_width] for NCW data format.
+ filters : tensor
+ A 3-D Tensor with the same type as value and shape [filter_width, output_channels, in_channels].
+ filter's in_channels dimension must match that of value.
+ output_shape : tensor
+ A 1-D Tensor, containing three elements, representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1 or 3. The number of entries by which the filter is moved right at each step.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NWC' and 'NCW' are supported.
+ dilations : list
+ An int or list of ints that has length 1 or 3 which defaults to 1.
+ The dilation factor for each dimension of input. If set to k > 1,
+ there will be k-1 skipped cells between each filter element on that dimension.
+ Dilations in the batch and depth dimensions must be 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ data_format, padding = preprocess_1d_format(data_format, padding)
+ outputs = tf.nn.conv1d_transpose(
+ input=input,
+ filters=filters,
+ output_shape=output_shape,
+ strides=strides,
+ padding=padding,
+ data_format=data_format,
+ dilations=dilations,
+ name=name,
+ )
+ return outputs
+
+
+class Conv2d_transpose(object):
+
+ def __init__(
+ self, strides, padding, data_format='NHWC', dilations=None, name=None, out_channel=None, k_size=None,
+ in_channels=None
+ ):
+ self.strides = strides
+ self.dilations = dilations
+ self.name = name
+ self.data_format, self.padding = preprocess_2d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ if self.data_format == 'NHWC':
+ h_axis, w_axis = 1, 2
+ else:
+ h_axis, w_axis = 2, 3
+
+ input_shape = input.shape.as_list()
+ filters_shape = filters.shape.as_list()
+ batch_size = input.shape[0]
+ input_h, input_w = input_shape[h_axis], input_shape[w_axis]
+ kernel_h, kernel_w = filters_shape[0], filters_shape[1]
+ output_channels = filters_shape[2]
+ dilations_h, dilations_w = 1, 1
+
+ if isinstance(self.strides, int):
+ strides_h = self.strides
+ strides_w = self.strides
+ else:
+ strides_list = list(self.strides)
+ if len(strides_list) != 4:
+ strides_h = strides_list[0]
+ strides_w = strides_list[1]
+ else:
+ strides_h = strides_list[h_axis]
+ strides_w = strides_list[w_axis]
+
+ if self.dilations is not None:
+ if isinstance(self.dilations, int):
+ dilations_h = self.dilations
+ dilations_w = self.dilations
+ else:
+ dilations_list = list(self.dilations)
+ if len(dilations_list) != 4:
+ dilations_h = dilations_list[0]
+ dilations_w = dilations_list[1]
+ else:
+ dilations_h = dilations_list[h_axis]
+ dilations_w = dilations_list[w_axis]
+
+ kernel_h = kernel_h + (kernel_h - 1) * (dilations_h - 1)
+ kernel_w = kernel_w + (kernel_w - 1) * (dilations_w - 1)
+
+ assert self.padding in {'SAME', 'VALID'}
+ if self.padding == 'VALID':
+ output_h = input_h * strides_h + max(kernel_h - strides_h, 0)
+ output_w = input_w * strides_w + max(kernel_w - strides_w, 0)
+ elif self.padding == 'SAME':
+ output_h = input_h * strides_h
+ output_w = input_w * strides_w
+
+ if self.data_format == 'NCHW':
+ out_shape = (batch_size, output_channels, output_h, output_w)
+ else:
+ out_shape = (batch_size, output_h, output_w, output_channels)
+
+ output_shape = tf.stack(out_shape)
+
+ outputs = tf.nn.conv2d_transpose(
+ input=input, filters=filters, output_shape=output_shape, strides=self.strides, padding=self.padding,
+ data_format=self.data_format, dilations=self.dilations, name=self.name
+ )
+ return outputs
+
+
+def conv2d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NHWC', dilations=None, name=None
+):
+ """
+ The transpose of conv2d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 4-D Tensor of type float and shape [batch, height, width, in_channels]
+ for NHWC data format or [batch, in_channels, height, width] for NCHW data format.
+ filters : tensor
+ A 4-D Tensor with the same type as input and shape [height, width,
+ output_channels, in_channels]. filter's in_channels dimension must match that of input.
+ output_shape : tensor
+ A 1-D Tensor representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1, 2 or 4. The stride of the sliding window for each dimension of input.
+ If a single value is given it is replicated in the H and W dimension.
+ By default the N and C dimensions are set to 0.
+ The dimension order is determined by the value of data_format, see below for details.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NHWC' and 'NCHW' are supported.
+ dilations : list
+ An int or list of ints that has length 1, 2 or 4, defaults to 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as input.
+ """
+
+ data_format, padding = preprocess_2d_format(data_format, padding)
+ outputs = tf.nn.conv2d_transpose(
+ input=input,
+ filters=filters,
+ output_shape=output_shape,
+ strides=strides,
+ padding=padding,
+ data_format=data_format,
+ dilations=dilations,
+ name=name,
+ )
+ return outputs
+
+
+class Conv3d_transpose(object):
+
+ def __init__(
+ self, strides, padding, data_format='NDHWC', dilations=None, name=None, out_channel=None, k_size=None,
+ in_channels=None
+ ):
+ self.strides = strides
+ self.dilations = dilations
+ self.name = name
+ self.out_channel = out_channel
+ self.data_format, self.padding = preprocess_3d_format(data_format, padding)
+
+ def __call__(self, input, filters):
+ if self.data_format == 'NDHWC':
+ d_axis, h_axis, w_axis = 1, 2, 3
+ else:
+ d_axis, h_axis, w_axis = 2, 3, 4
+
+ input_shape = input.shape.as_list()
+ filters_shape = filters.shape.as_list()
+ batch_size = input_shape[0]
+ input_d, input_h, input_w = input_shape[d_axis], input_shape[h_axis], input_shape[w_axis]
+ kernel_d, kernel_h, kernel_w = filters_shape[0], filters_shape[1], filters_shape[2]
+ dilations_d, dilations_h, dilations_w = 1, 1, 1
+
+ if isinstance(self.strides, int):
+ strides_d, strides_h, strides_w = self.strides
+ else:
+ strides_list = list(self.strides)
+ if len(strides_list) != 5:
+ strides_d, strides_h, strides_w = \
+ strides_list[0], \
+ strides_list[1], \
+ strides_list[2]
+ else:
+ strides_d, strides_h, strides_w = \
+ strides_list[d_axis], \
+ strides_list[h_axis], \
+ strides_list[w_axis]
+
+ if self.dilations is not None:
+ if isinstance(self.dilations, int):
+ dilations_d, dilations_h, dilations_w = self.dilations
+ else:
+ dilations_list = list(self.dilations)
+ if len(dilations_list) != 5:
+ dilations_d, dilations_h, dilations_w = \
+ dilations_list[0], \
+ dilations_list[1], \
+ dilations_list[2]
+ else:
+ dilations_d, dilations_h, dilations_w = \
+ dilations_list[d_axis],\
+ dilations_list[h_axis], \
+ dilations_list[w_axis]
+
+ assert self.padding in {'VALID', 'SAME'}
+
+ kernel_d = kernel_d + (kernel_d - 1) * (dilations_d - 1)
+ kernel_h = kernel_h + (kernel_h - 1) * (dilations_h - 1)
+ kernel_w = kernel_w + (kernel_w - 1) * (dilations_w - 1)
+
+ if self.padding == 'VALID':
+ output_d = input_d * strides_d + max(kernel_d - strides_d, 0)
+ output_h = input_h * strides_h + max(kernel_h - strides_h, 0)
+ output_w = input_w * strides_w + max(kernel_w - strides_w, 0)
+ elif self.padding == 'SAME':
+ output_d = input_d * strides_d
+ output_h = input_h * strides_h
+ output_w = input_w * strides_w
+
+ if self.data_format == 'NDHWC':
+ output_shape = (batch_size, output_d, output_h, output_w, self.out_channel)
+ else:
+ output_shape = (batch_size, self.out_channel, output_d, output_h, output_w)
+
+ output_shape = tf.stack(output_shape)
+ outputs = tf.nn.conv3d_transpose(
+ input=input, filters=filters, output_shape=output_shape, strides=self.strides, padding=self.padding,
+ data_format=self.data_format, dilations=self.dilations, name=self.name
+ )
+
+ return outputs
+
+
+def conv3d_transpose(
+ input, filters, output_shape, strides, padding='SAME', data_format='NDHWC', dilations=None, name=None
+):
+ """
+ The transpose of conv3d.
+
+ Parameters
+ ----------
+ input : tensor
+ A 5-D Tensor of type float and shape [batch, height, width, in_channels] for
+ NHWC data format or [batch, in_channels, height, width] for NCHW data format.
+ filters : tensor
+ A 5-D Tensor with the same type as value and shape [height, width, output_channels, in_channels].
+ filter's in_channels dimension must match that of value.
+ output_shape : tensor
+ A 1-D Tensor representing the output shape of the deconvolution op.
+ strides : list
+ An int or list of ints that has length 1, 3 or 5.
+ padding : string
+ 'VALID' or 'SAME'. The padding algorithm. See the "returns" section of tf.ops.convolution for details.
+ data_format : string
+ 'NDHWC' and 'NCDHW' are supported.
+ dilations : list of ints
+ An int or list of ints that has length 1, 3 or 5, defaults to 1.
+ name : string
+ Optional name for the returned tensor.
+
+ Returns
+ -------
+ A Tensor with the same type as value.
+ """
+
+ data_format, padding = preprocess_3d_format(data_format, padding)
+ outputs = tf.nn.conv3d_transpose(
+ input=input, filters=filters, output_shape=output_shape, strides=strides, padding=padding,
+ data_format=data_format, dilations=dilations, name=name
+ )
+ return outputs
+
+
+def depthwise_conv2d(input, filters, strides, padding='SAME', data_format='NHWC', dilations=None, name=None):
+ """
+ Depthwise 2-D convolution.
+
+ Parameters
+ ----------
+ input : tensor
+ 4-D with shape according to data_format.
+ filters : tensor
+ 4-D with shape [filter_height, filter_width, in_channels, channel_multiplier].
+ strides : tuple
+ 1-D of size 4. The stride of the sliding window for each dimension of input.
+ padding : string
+ 'VALID' or 'SAME'
+ data_format : string
+ "NHWC" (default) or "NCHW".
+ dilations : tuple
+ The dilation rate in which we sample input values across the height and width dimensions in atrous convolution.
+ If it is greater than 1, then all values of strides must be 1.
+ name : string
+ A name for this operation (optional).
+
+ Returns
+ -------
+ A 4-D Tensor with shape according to data_format.
+ """
+
+ data_format, padding = preprocess_2d_format(data_format, padding)
+ outputs = tf.nn.depthwise_conv2d(
+ input=input,
+ filter=filters,
+ strides=strides,
+ padding=padding,
+ data_format=data_format,
+ dilations=dilations,
+ name=name,
+ )
+ return outputs
+
+
+def _to_channel_first_bias(b):
+ """Reshape [c] to [c, 1, 1]."""
+ channel_size = int(b.shape[0])
+ new_shape = (channel_size, 1, 1)
+ return tf.reshape(b, new_shape)
+
+
+def _bias_scale(x, b, data_format):
+ """The multiplication counter part of tf.nn.bias_add."""
+ if data_format == 'NHWC':
+ return x * b
+ elif data_format == 'NCHW':
+ return x * _to_channel_first_bias(b)
+ else:
+ raise ValueError('invalid data_format: %s' % data_format)
+
+
+def _bias_add(x, b, data_format):
+ """Alternative implementation of tf.nn.bias_add which is compatiable with tensorRT."""
+ if data_format == 'NHWC':
+ return tf.add(x, b)
+ elif data_format == 'NCHW':
+ return tf.add(x, _to_channel_first_bias(b))
+ else:
+ raise ValueError('invalid data_format: %s' % data_format)
+
+
+def batch_normalization(x, mean, variance, offset, scale, variance_epsilon, data_format, name=None):
+ """Data Format aware version of tf.nn.batch_normalization."""
+ if data_format == 'channels_last':
+ mean = tf.reshape(mean, [1] * (len(x.shape) - 1) + [-1])
+ variance = tf.reshape(variance, [1] * (len(x.shape) - 1) + [-1])
+ offset = tf.reshape(offset, [1] * (len(x.shape) - 1) + [-1])
+ scale = tf.reshape(scale, [1] * (len(x.shape) - 1) + [-1])
+ elif data_format == 'channels_first':
+ mean = tf.reshape(mean, [1] + [-1] + [1] * (len(x.shape) - 2))
+ variance = tf.reshape(variance, [1] + [-1] + [1] * (len(x.shape) - 2))
+ offset = tf.reshape(offset, [1] + [-1] + [1] * (len(x.shape) - 2))
+ scale = tf.reshape(scale, [1] + [-1] + [1] * (len(x.shape) - 2))
+ else:
+ raise ValueError('invalid data_format: %s' % data_format)
+
+ with ops.name_scope(name, 'batchnorm', [x, mean, variance, scale, offset]):
+ inv = math_ops.rsqrt(variance + variance_epsilon)
+ if scale is not None:
+ inv *= scale
+
+ a = math_ops.cast(inv, x.dtype)
+ b = math_ops.cast(offset - mean * inv if offset is not None else -mean * inv, x.dtype)
+ # Return a * x + b with customized data_format.
+ # Currently TF doesn't have bias_scale, and tensorRT has bug in converting tf.nn.bias_add
+ # So we reimplemted them to allow make the model work with tensorRT.
+ # See https://github.com/tensorlayer/openpose-plus/issues/75 for more details.
+ # df = {'channels_first': 'NCHW', 'channels_last': 'NHWC'}
+ # return _bias_add(_bias_scale(x, a, df[data_format]), b, df[data_format])
+ return a * x + b
+
+
+class BatchNorm(object):
+ """
+ The :class:`BatchNorm` is a batch normalization layer for both fully-connected and convolution outputs.
+ See ``tf.nn.batch_normalization`` and ``tf.nn.moments``.
+
+ Parameters
+ ----------
+ decay : float
+ A decay factor for `ExponentialMovingAverage`.
+ Suggest to use a large value for large dataset.
+ epsilon : float
+ Eplison.
+ act : activation function
+ The activation function of this layer.
+ is_train : boolean
+ Is being used for training or inference.
+ beta_init : initializer or None
+ The initializer for initializing beta, if None, skip beta.
+ Usually you should not skip beta unless you know what happened.
+ gamma_init : initializer or None
+ The initializer for initializing gamma, if None, skip gamma.
+ When the batch normalization layer is use instead of 'biases', or the next layer is linear, this can be
+ disabled since the scaling can be done by the next layer. see `Inception-ResNet-v2 `__
+ moving_mean_init : initializer or None
+ The initializer for initializing moving mean, if None, skip moving mean.
+ moving_var_init : initializer or None
+ The initializer for initializing moving var, if None, skip moving var.
+ num_features: int
+ Number of features for input tensor. Useful to build layer if using BatchNorm1d, BatchNorm2d or BatchNorm3d,
+ but should be left as None if using BatchNorm. Default None.
+ data_format : str
+ channels_last 'channel_last' (default) or channels_first.
+ name : None or str
+ A unique layer name.
+
+ Examples
+ ---------
+ With TensorLayer
+
+ >>> net = tl.layers.Input([None, 50, 50, 32], name='input')
+ >>> net = tl.layers.BatchNorm()(net)
+
+ Notes
+ -----
+ The :class:`BatchNorm` is universally suitable for 3D/4D/5D input in static model, but should not be used
+ in dynamic model where layer is built upon class initialization. So the argument 'num_features' should only be used
+ for subclasses :class:`BatchNorm1d`, :class:`BatchNorm2d` and :class:`BatchNorm3d`. All the three subclasses are
+ suitable under all kinds of conditions.
+
+ References
+ ----------
+ - `Source `__
+ - `stackoverflow `__
+
+ """
+
+ def __init__(
+ self, decay=0.9, epsilon=0.00001, beta=None, gamma=None, moving_mean=None, moving_var=None, num_features=None,
+ data_format='channels_last', is_train=False
+ ):
+ self.decay = decay
+ self.epsilon = epsilon
+ self.data_format = data_format
+ self.beta = beta
+ self.gamma = gamma
+ self.moving_mean = moving_mean
+ self.moving_var = moving_var
+ self.num_features = num_features
+ self.is_train = is_train
+ self.axes = None
+
+ if self.decay < 0.0 or 1.0 < self.decay:
+ raise ValueError("decay should be between 0 to 1")
+
+ def _get_param_shape(self, inputs_shape):
+ if self.data_format == 'channels_last':
+ axis = -1
+ elif self.data_format == 'channels_first':
+ axis = 1
+ else:
+ raise ValueError('data_format should be either %s or %s' % ('channels_last', 'channels_first'))
+
+ channels = inputs_shape[axis]
+ params_shape = [channels]
+
+ return params_shape
+
+ def _check_input_shape(self, inputs):
+ if inputs.ndim <= 1:
+ raise ValueError('expected input at least 2D, but got {}D input'.format(inputs.ndim))
+
+ def __call__(self, inputs):
+ self._check_input_shape(inputs)
+ self.channel_axis = len(inputs.shape) - 1 if self.data_format == 'channels_last' else 1
+ if self.axes is None:
+ self.axes = [i for i in range(len(inputs.shape)) if i != self.channel_axis]
+
+ mean, var = tf.nn.moments(inputs, self.axes, keepdims=False)
+ if self.is_train:
+ # update moving_mean and moving_var
+ self.moving_mean = moving_averages.assign_moving_average(
+ self.moving_mean, mean, self.decay, zero_debias=False
+ )
+ self.moving_var = moving_averages.assign_moving_average(self.moving_var, var, self.decay, zero_debias=False)
+ outputs = batch_normalization(inputs, mean, var, self.beta, self.gamma, self.epsilon, self.data_format)
+ else:
+ outputs = batch_normalization(
+ inputs, self.moving_mean, self.moving_var, self.beta, self.gamma, self.epsilon, self.data_format
+ )
+
+ return outputs
diff --git a/tensorlayer/cli/__init__.py b/tensorlayer/cli/__init__.py
new file mode 100644
index 0000000..e7522d7
--- /dev/null
+++ b/tensorlayer/cli/__init__.py
@@ -0,0 +1,3 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""The tensorlayer.cli module provides a command-line tool for some common tasks."""
diff --git a/tensorlayer/cli/__main__.py b/tensorlayer/cli/__main__.py
new file mode 100644
index 0000000..838038b
--- /dev/null
+++ b/tensorlayer/cli/__main__.py
@@ -0,0 +1,17 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import argparse
+
+from tensorlayer.cli import train
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(prog='tl')
+ subparsers = parser.add_subparsers(dest='cmd')
+ train_parser = subparsers.add_parser('train', help='train a model using multiple local GPUs or CPUs.')
+ train.build_arg_parser(train_parser)
+ args = parser.parse_args()
+ if args.cmd == 'train':
+ train.main(args)
+ else:
+ parser.print_help()
diff --git a/tensorlayer/cli/train.py b/tensorlayer/cli/train.py
new file mode 100644
index 0000000..d63b7f5
--- /dev/null
+++ b/tensorlayer/cli/train.py
@@ -0,0 +1,171 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+tl train
+========
+
+(Alpha release - usage might change later)
+
+The tensorlayer.cli.train module provides the ``tl train`` subcommand.
+It helps the user bootstrap a TensorFlow/TensorLayer program for distributed training
+using multiple GPU cards or CPUs on a computer.
+
+You need to first setup the `CUDA_VISIBLE_DEVICES `_
+to tell ``tl train`` which GPUs are available. If the CUDA_VISIBLE_DEVICES is not given,
+``tl train`` would try best to discover all available GPUs.
+
+In distribute training, each TensorFlow program needs a TF_CONFIG environment variable to describe
+the cluster. It also needs a master daemon to
+monitor all trainers. ``tl train`` is responsible
+for automatically managing these two tasks.
+
+Usage
+-----
+
+tl train [-h] [-p NUM_PSS] [-c CPU_TRAINERS] [args [args ...]]
+
+.. code-block:: bash
+
+ # example of using GPU 0 and 1 for training mnist
+ CUDA_VISIBLE_DEVICES="0,1"
+ tl train example/tutorial_mnist_distributed.py
+
+ # example of using CPU trainers for inception v3
+ tl train -c 16 example/tutorial_imagenet_inceptionV3_distributed.py
+
+ # example of using GPU trainers for inception v3 with customized arguments
+ # as CUDA_VISIBLE_DEVICES is not given, tl would try to discover all available GPUs
+ tl train example/tutorial_imagenet_inceptionV3_distributed.py -- --batch_size 16
+
+
+Command-line Arguments
+----------------------
+
+- ``file``: python file path.
+
+- ``NUM_PSS`` : The number of parameter servers.
+
+- ``CPU_TRAINERS``: The number of CPU trainers.
+
+ It is recommended that ``NUM_PSS + CPU_TRAINERS <= cpu count``
+
+- ``args``: Any parameter after ``--`` would be passed to the python program.
+
+
+Notes
+-----
+A parallel training program would require multiple parameter servers
+to help parallel trainers to exchange intermediate gradients.
+The best number of parameter servers is often proportional to the
+size of your model as well as the number of CPUs available.
+You can control the number of parameter servers using the ``-p`` parameter.
+
+If you have a single computer with massive CPUs, you can use the ``-c`` parameter
+to enable CPU-only parallel training.
+The reason we are not supporting GPU-CPU co-training is because GPU and
+CPU are running at different speeds. Using them together in training would
+incur stragglers.
+
+"""
+
+import argparse
+import json
+import multiprocessing
+import os
+import platform
+import re
+import subprocess
+import sys
+
+PORT_BASE = 10000
+
+
+def _get_gpu_ids():
+ if 'CUDA_VISIBLE_DEVICES' in os.environ:
+ return [int(x) for x in os.environ.get('CUDA_VISIBLE_DEVICES', '').split(',')]
+ if platform.system() in ['Darwin', 'Linux']:
+ return [int(d.replace('nvidia', '')) for d in os.listdir('/dev') if re.match('^nvidia\d+$', d)]
+ else:
+ print('Please set CUDA_VISIBLE_DEVICES (see http://acceleware.com/blog/cudavisibledevices-masking-gpus)')
+ return []
+
+
+GPU_IDS = _get_gpu_ids()
+
+
+def create_tf_config(cluster_spec, task_type, task_index):
+ return {
+ 'cluster': cluster_spec,
+ 'task': {
+ 'type': task_type,
+ 'index': task_index
+ },
+ }
+
+
+def create_tf_jobs(cluster_spec, prog, args):
+ gpu_assignment = dict((('worker', idx), gpu_idx) for (idx, gpu_idx) in enumerate(GPU_IDS))
+ for job_type in cluster_spec:
+ for task_index in range(len(cluster_spec[job_type])):
+ new_env = os.environ.copy()
+ new_env.update(
+ {
+ 'CUDA_VISIBLE_DEVICES': str(gpu_assignment.get((job_type, task_index), '')),
+ 'TF_CONFIG': json.dumps(create_tf_config(cluster_spec, job_type, task_index)),
+ }
+ )
+ yield subprocess.Popen(['python3', prog] + args, env=new_env)
+
+
+def validate_arguments(args):
+ if args.num_pss < 1:
+ print('Value error: must have ore than one parameter servers.')
+ exit(1)
+
+ if not GPU_IDS:
+ num_cpus = multiprocessing.cpu_count()
+ if args.cpu_trainers > num_cpus:
+ print('Value error: there are %s available CPUs but you are requiring %s.' % (num_cpus, args.cpu_trainers))
+ exit(1)
+
+ if not os.path.isfile(args.file):
+ print('Value error: model trainning file does not exist')
+ exit(1)
+
+
+def main(args):
+ validate_arguments(args)
+ num_workers = len(GPU_IDS) if GPU_IDS else args.cpu_trainers
+ print('Using program %s with args %s' % (args.file, ' '.join(args.args)))
+ print('Using %d workers, %d parameter servers, %d GPUs.' % (num_workers, args.num_pss, len(GPU_IDS)))
+ cluster_spec = {
+ 'ps': ['localhost: %d' % (PORT_BASE + i) for i in range(args.num_pss)],
+ 'worker': ['localhost: %d' % (PORT_BASE + args.num_pss + i) for i in range(num_workers)]
+ }
+ processes = list(create_tf_jobs(cluster_spec, args.file, args.args))
+ try:
+ print('Press ENTER to exit the training ...')
+ sys.stdin.readline()
+ except KeyboardInterrupt: # https://docs.python.org/3/library/exceptions.html#KeyboardInterrupt
+ print('Keyboard interrupt received')
+ finally:
+ print('stopping all subprocesses ...')
+ for p in processes:
+ p.kill()
+ for p in processes:
+ p.wait()
+ print('END')
+
+
+def build_arg_parser(parser):
+ parser.add_argument('-p', '--pss', dest='num_pss', type=int, default=1, help='number of parameter servers')
+ parser.add_argument('-c', '--cpu_trainers', dest='cpu_trainers', type=int, default=1, help='number of CPU trainers')
+ parser.add_argument('file', help='model trainning file path')
+ parser.add_argument('args', nargs='*', type=str, help='arguments to ')
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ build_arg_parser(parser)
+ args = parser.parse_args()
+ main(args)
diff --git a/tensorlayer/cost/__init__.py b/tensorlayer/cost/__init__.py
new file mode 100644
index 0000000..3ca7c2c
--- /dev/null
+++ b/tensorlayer/cost/__init__.py
@@ -0,0 +1,15 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from tensorlayer.backend import BACKEND
+
+if BACKEND == 'tensorflow':
+ from .tensorflow_cost import *
+elif BACKEND == 'mindspore':
+ from .mindspore_cost import *
+elif BACKEND == 'dragon':
+ pass
+elif BACKEND == 'paddle':
+ from .paddle_cost import *
+else:
+ raise NotImplementedError("This backend is not supported")
diff --git a/tensorlayer/cost/mindspore_cost.py b/tensorlayer/cost/mindspore_cost.py
new file mode 100644
index 0000000..694c5fc
--- /dev/null
+++ b/tensorlayer/cost/mindspore_cost.py
@@ -0,0 +1,763 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from mindspore import nn
+from mindspore.nn import Cell
+import mindspore.ops as P
+
+__all__ = [
+ 'cross_entropy',
+ 'sigmoid_cross_entropy',
+ 'binary_cross_entropy',
+ 'mean_squared_error',
+ 'normalized_mean_square_error',
+ 'absolute_difference_error',
+ 'dice_coe',
+ 'dice_hard_coe',
+ 'iou_coe',
+ 'cross_entropy_seq',
+ 'cross_entropy_seq_with_mask',
+ 'cosine_similarity',
+ 'li_regularizer',
+ 'lo_regularizer',
+ 'maxnorm_regularizer',
+ 'maxnorm_o_regularizer',
+ 'maxnorm_i_regularizer',
+]
+
+cross_entropy = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
+
+
+def sigmoid_cross_entropy(output, target, name=None):
+ """Sigmoid cross-entropy operation, see ``tf.ops.sigmoid_cross_entropy_with_logits``.
+
+ Parameters
+ ----------
+ output : Tensor
+ A batch of distribution with shape: [batch_size, num of classes].
+ target : Tensor
+ A batch of index with shape: [batch_size, ].
+ name : string
+ Name of this loss.
+
+ """
+ outputs = P.ReduceMean(cross_entropy(output, target))
+ return outputs
+
+
+def binary_cross_entropy(output, target, epsilon=1e-8, name='bce_loss'):
+ """Binary cross entropy operation.
+
+ Parameters
+ ----------
+ output : Tensor
+ Tensor with type of `float32` or `float64`.
+ target : Tensor
+ The target distribution, format the same with `output`.
+ epsilon : float
+ A small value to avoid output to be zero.
+ name : str
+ An optional name to attach to this function.
+
+ References
+ -----------
+ - `ericjang-DRAW `__
+
+ """
+
+ # return tf.reduce_mean(
+ # tf.reduce_sum(
+ # -(target * tf.math.log(output + epsilon) + (1. - target) * tf.math.log(1. - output + epsilon)), axis=1
+ # ), name=name
+ # )
+ raise NotImplementedError("Not Implemented.")
+
+
+def mean_squared_error(output, target, is_mean=False, axis=-1, name="mean_squared_error"):
+ """Return the TensorFlow expression of mean-square-error (L2) of two batch of data.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data.
+ - If False, use ``tf.reduce_sum`` (default).
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ References
+ ------------
+ - `Wiki Mean Squared Error `__
+
+ """
+ # with tf.name_scope(name):
+ # if len(output.shape) == 2: # [batch_size, n_feature]
+ # axis = 1
+ # elif len(output.shape) == 3: # [batch_size, w, h]
+ # axis = [1, 2]
+ # elif len(output.shape) == 4: # [batch_size, w, h, c]
+ # axis = [1, 2, 3]
+ # else:
+ # raise Exception("Unknow dimension")
+
+ return nn.MSELoss()(output, target)
+
+
+def normalized_mean_square_error(output, target, axis=-1, name="normalized_mean_squared_error_loss"):
+ """Return the TensorFlow expression of normalized mean-square-error of two distributions.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ """
+ # with tf.name_scope("normalized_mean_squared_error_loss"):
+ # nmse_a = tf.sqrt(tf.reduce_sum(tf.math.squared_difference(output, target), axis=axis))
+ # nmse_b = tf.sqrt(tf.reduce_sum(tf.square(target), axis=axis))
+ # nmse = tf.reduce_mean(nmse_a / nmse_b, name=name)
+ raise NotImplementedError("Not Implemented.")
+
+
+def absolute_difference_error(output, target, is_mean=False, axis=-1, name="absolute_difference_error_loss"):
+ """Return the TensorFlow expression of absolute difference error (L1) of two batch of data.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data.
+ - If False, use ``tf.reduce_sum`` (default).
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ """
+
+ # if is_mean:
+ # loss = tf.reduce_mean(tf.reduce_mean(tf.abs(output - target), axis), name=name)
+ # else:
+ # loss = tf.reduce_mean(tf.reduce_sum(tf.abs(output - target), axis), name=name)
+ raise NotImplementedError("Not Implemented.")
+
+
+def dice_coe(output, target, loss_type='jaccard', axis=(1, 2, 3), smooth=1e-5):
+ """Soft dice (Sørensen or Jaccard) coefficient for comparing the similarity
+ of two batch of data, usually be used for binary image segmentation
+ i.e. labels are binary. The coefficient between 0 to 1, 1 means totally match.
+
+ Parameters
+ -----------
+ output : Tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : Tensor
+ The target distribution, format the same with `output`.
+ loss_type : str
+ ``jaccard`` or ``sorensen``, default is ``jaccard``.
+ axis : tuple of int
+ All dimensions are reduced, default ``[1,2,3]``.
+ smooth : float
+ This small value will be added to the numerator and denominator.
+ - If both output and target are empty, it makes sure dice is 1.
+ - If either output or target are empty (all pixels are background), dice = ```smooth/(small_value + smooth)``, then if smooth is very small, dice close to 0 (even the image values lower than the threshold), so in this case, higher smooth can have a higher dice.
+
+ Examples
+ ---------
+ >>> import tensorlayer as tl
+ >>> outputs = tl.act.pixel_wise_softmax(outputs)
+ >>> dice_loss = 1 - tl.cost.dice_coe(outputs, y_)
+
+ References
+ -----------
+ - `Wiki-Dice `__
+
+ """
+ # inse = tf.reduce_sum(output * target, axis=axis)
+ # if loss_type == 'jaccard':
+ # l = tf.reduce_sum(output * output, axis=axis)
+ # r = tf.reduce_sum(target * target, axis=axis)
+ # elif loss_type == 'sorensen':
+ # l = tf.reduce_sum(output, axis=axis)
+ # r = tf.reduce_sum(target, axis=axis)
+ # else:
+ # raise Exception("Unknow loss_type")
+ # dice = (2. * inse + smooth) / (l + r + smooth)
+ # ##
+ # dice = tf.reduce_mean(dice, name='dice_coe')
+ raise NotImplementedError("Not Implemented.")
+
+
+def dice_hard_coe(output, target, threshold=0.5, axis=(1, 2, 3), smooth=1e-5):
+ """Non-differentiable Sørensen–Dice coefficient for comparing the similarity
+ of two batch of data, usually be used for binary image segmentation i.e. labels are binary.
+ The coefficient between 0 to 1, 1 if totally match.
+
+ Parameters
+ -----------
+ output : tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : tensor
+ The target distribution, format the same with `output`.
+ threshold : float
+ The threshold value to be true.
+ axis : tuple of integer
+ All dimensions are reduced, default ``(1,2,3)``.
+ smooth : float
+ This small value will be added to the numerator and denominator, see ``dice_coe``.
+
+ References
+ -----------
+ - `Wiki-Dice `__
+
+ """
+ # output = tf.cast(output > threshold, dtype=tf.float32)
+ # target = tf.cast(target > threshold, dtype=tf.float32)
+ # inse = tf.reduce_sum(tf.multiply(output, target), axis=axis)
+ # l = tf.reduce_sum(output, axis=axis)
+ # r = tf.reduce_sum(target, axis=axis)
+ # hard_dice = (2. * inse + smooth) / (l + r + smooth)
+ # ##
+ # hard_dice = tf.reduce_mean(hard_dice, name='hard_dice')
+ raise NotImplementedError("Not Implemented.")
+
+
+def iou_coe(output, target, threshold=0.5, axis=(1, 2, 3), smooth=1e-5):
+ """Non-differentiable Intersection over Union (IoU) for comparing the
+ similarity of two batch of data, usually be used for evaluating binary image segmentation.
+ The coefficient between 0 to 1, and 1 means totally match.
+
+ Parameters
+ -----------
+ output : tensor
+ A batch of distribution with shape: [batch_size, ....], (any dimensions).
+ target : tensor
+ The target distribution, format the same with `output`.
+ threshold : float
+ The threshold value to be true.
+ axis : tuple of integer
+ All dimensions are reduced, default ``(1,2,3)``.
+ smooth : float
+ This small value will be added to the numerator and denominator, see ``dice_coe``.
+
+ Notes
+ ------
+ - IoU cannot be used as training loss, people usually use dice coefficient for training, IoU and hard-dice for evaluating.
+
+ """
+ # pre = tf.cast(output > threshold, dtype=tf.float32)
+ # truth = tf.cast(target > threshold, dtype=tf.float32)
+ # inse = tf.reduce_sum(tf.multiply(pre, truth), axis=axis) # AND
+ # union = tf.reduce_sum(tf.cast(tf.add(pre, truth) >= 1, dtype=tf.float32), axis=axis) # OR
+ # batch_iou = (inse + smooth) / (union + smooth)
+ # iou = tf.reduce_mean(batch_iou, name='iou_coe')
+ raise NotImplementedError("Not Implemented.")
+
+
+def sequence_loss_by_example(
+ logits, targets, weights, average_across_timesteps=True, softmax_loss_function=None, name=None
+):
+ """Weighted cross-entropy loss for a sequence of logits (per example). see original tensorflow code :
+
+
+ Parameters
+ ----------
+ logits: List
+ List of 2D Tensors of shape [batch_size x num_decoder_symbols].
+ targets: List
+ List of 1D batch-sized int32 Tensors of the same length as logits.
+ weights: List
+ List of 1D batch-sized float-Tensors of the same length as logits.
+ average_across_timesteps: Boolean
+ If set, divide the returned cost by the total label weight.
+ softmax_loss_function: None or Function
+ Function (labels, logits) -> loss-batch to be used instead of the standard softmax (the default if this is None).
+ **Note that to avoid confusion, it is required for the function to accept named arguments.**
+ name: None or str
+ Optional name for this operation, default: "sequence_loss_by_example".
+
+ Returns
+ -------
+ 1D batch-sized float Tensor: The log-perplexity for each sequence.
+
+ Raises
+ ------
+ ValueError: If len(logits) is different from len(targets) or len(weights).
+
+ """
+ # if len(targets) != len(logits) or len(weights) != len(logits):
+ # raise ValueError(
+ # "Lengths of logits, weights, and targets must be the same "
+ # "%d, %d, %d." % (len(logits), len(weights), len(targets))
+ # )
+ # with ops.name_scope(name, "sequence_loss_by_example", logits + targets + weights):
+ # log_perp_list = []
+ # for logit, target, weight in zip(logits, targets, weights):
+ # if softmax_loss_function is None:
+ # # TODO(irving,ebrevdo): This reshape is needed because
+ # # sequence_loss_by_example is called with scalars sometimes, which
+ # # violates our general scalar strictness policy.
+ # target = array_ops.reshape(target, [-1])
+ # crossent = nn_ops.sparse_softmax_cross_entropy_with_logits(labels=target, logits=logit)
+ # else:
+ # crossent = softmax_loss_function(labels=target, logits=logit)
+ # log_perp_list.append(crossent * weight)
+ # log_perps = math_ops.add_n(log_perp_list)
+ # if average_across_timesteps:
+ # total_size = math_ops.add_n(weights)
+ # total_size += 1e-12 # Just to avoid division by 0 for all-0 weights.
+ # log_perps /= total_size
+ raise NotImplementedError("Not Implemented.")
+
+
+def cross_entropy_seq(logits, target_seqs, batch_size=None):
+ """Returns the expression of cross-entropy of two sequences, implement
+ softmax internally. Normally be used for fixed length RNN outputs, see `PTB example `__.
+
+ Parameters
+ ----------
+ logits : Tensor
+ 2D tensor with shape of `[batch_size * n_steps, n_classes]`.
+ target_seqs : Tensor
+ The target sequence, 2D tensor `[batch_size, n_steps]`, if the number of step is dynamic, please use ``tl.cost.cross_entropy_seq_with_mask`` instead.
+ batch_size : None or int.
+ Whether to divide the cost by batch size.
+ - If integer, the return cost will be divided by `batch_size`.
+ - If None (default), the return cost will not be divided by anything.
+
+ Examples
+ --------
+ >>> import tensorlayer as tl
+ >>> # see `PTB example `__.for more details
+ >>> # outputs shape : (batch_size * n_steps, n_classes)
+ >>> # targets shape : (batch_size, n_steps)
+ >>> cost = tl.cost.cross_entropy_seq(outputs, targets)
+
+ """
+ # sequence_loss_by_example_fn = sequence_loss_by_example
+ #
+ # loss = sequence_loss_by_example_fn(
+ # [logits], [tf.reshape(target_seqs, [-1])], [tf.ones_like(tf.reshape(target_seqs, [-1]), dtype=tf.float32)]
+ # )
+ # # [tf.ones([batch_size * num_steps])])
+ # cost = tf.reduce_sum(loss) # / batch_size
+ # if batch_size is not None:
+ # cost = cost / batch_size
+ raise NotImplementedError("Not Implemented.")
+
+
+def cross_entropy_seq_with_mask(logits, target_seqs, input_mask, return_details=False, name=None):
+ """Returns the expression of cross-entropy of two sequences, implement
+ softmax internally. Normally be used for Dynamic RNN with Synced sequence input and output.
+
+ Parameters
+ -----------
+ logits : Tensor
+ 2D tensor with shape of [batch_size * ?, n_classes], `?` means dynamic IDs for each example.
+ - Can be get from `DynamicRNNLayer` by setting ``return_seq_2d`` to `True`.
+ target_seqs : Tensor
+ int of tensor, like word ID. [batch_size, ?], `?` means dynamic IDs for each example.
+ input_mask : Tensor
+ The mask to compute loss, it has the same size with `target_seqs`, normally 0 or 1.
+ return_details : boolean
+ Whether to return detailed losses.
+ - If False (default), only returns the loss.
+ - If True, returns the loss, losses, weights and targets (see source code).
+
+ Examples
+ --------
+ >>> import tensorlayer as tl
+ >>> import tensorflow as tf
+ >>> import numpy as np
+ >>> batch_size = 64
+ >>> vocab_size = 10000
+ >>> embedding_size = 256
+ >>> ni = tl.layers.Input([batch_size, None], dtype=tf.int64)
+ >>> net = tl.layers.Embedding(
+ ... vocabulary_size = vocab_size,
+ ... embedding_size = embedding_size,
+ ... name = 'seq_embedding')(ni)
+ >>> net = tl.layers.RNN(
+ ... cell =tf.keras.layers.LSTMCell(units=embedding_size, dropout=0.1),
+ ... return_seq_2d = True,
+ ... name = 'dynamicrnn')(net)
+ >>> net = tl.layers.Dense(n_units=vocab_size, name="output")(net)
+ >>> model = tl.models.Model(inputs=ni, outputs=net)
+ >>> input_seqs = np.random.randint(0, 10, size=(batch_size, 10), dtype=np.int64)
+ >>> target_seqs = np.random.randint(0, 10, size=(batch_size, 10), dtype=np.int64)
+ >>> input_mask = np.random.randint(0, 2, size=(batch_size, 10), dtype=np.int64)
+ >>> outputs = model(input_seqs, is_train=True)
+ >>> loss = tl.cost.cross_entropy_seq_with_mask(outputs, target_seqs, input_mask)
+
+ """
+ # targets = tf.reshape(target_seqs, [-1]) # to one vector
+ # weights = tf.cast(tf.reshape(input_mask, [-1]), dtype=tf.float32) # to one vector like targets
+ # losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets, name=name) * weights
+ # # losses = tf.reduce_mean(tf.ops.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets, name=name)) # for TF1.0 and others
+ #
+ # loss = tf.divide(
+ # tf.reduce_sum(losses), # loss from mask. reduce_sum before element-wise mul with mask !!
+ # tf.reduce_sum(weights),
+ # name="seq_loss_with_mask"
+ # )
+ #
+ # if return_details:
+ # return loss, losses, weights, targets
+ # else:
+ # return loss
+ raise NotImplementedError("Not Implemented.")
+
+
+def cosine_similarity(v1, v2):
+ """Cosine similarity [-1, 1].
+
+ Parameters
+ ----------
+ v1, v2 : Tensor
+ Tensor with the same shape [batch_size, n_feature].
+
+ References
+ ----------
+ - `Wiki `__.
+
+ """
+
+ # return tf.reduce_sum(tf.multiply(v1, v2), 1) / \
+ # (tf.sqrt(tf.reduce_sum(tf.multiply(v1, v1), 1)) *
+ # tf.sqrt(tf.reduce_sum(tf.multiply(v2, v2), 1)))
+ raise NotImplementedError("Not Implemented.")
+
+
+# Regularization Functions
+def li_regularizer(scale, scope=None):
+ """Li regularization removes the neurons of previous layer. The `i` represents `inputs`.
+ Returns a function that can be used to apply group li regularization to weights.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+ scope: str
+ An optional scope name for this function.
+
+ Returns
+ --------
+ A function with signature `li(weights, name=None)` that apply Li regularization.
+
+ Raises
+ ------
+ ValueError : if scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ # if isinstance(scale, numbers.Integral):
+ # raise ValueError('scale cannot be an integer: %s' % scale)
+ # if isinstance(scale, numbers.Real):
+ # if scale < 0.:
+ # raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ # if scale >= 1.:
+ # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' % scale)
+ # if scale == 0.:
+ # logging.info('Scale of 0 disables regularizer.')
+ # return lambda _, name=None: None
+ #
+ # def li(weights):
+ # """Applies li regularization to weights."""
+ # with tf.name_scope('li_regularizer') as scope:
+ # my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ # # if tf.__version__ <= '0.12':
+ # # standard_ops_fn = standard_ops.mul
+ # # else:
+ # standard_ops_fn = standard_ops.multiply
+ # return standard_ops_fn(
+ # my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))),
+ # name=scope
+ # )
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def lo_regularizer(scale):
+ """Lo regularization removes the neurons of current layer. The `o` represents `outputs`
+ Returns a function that can be used to apply group lo regularization to weights.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ -------
+ A function with signature `lo(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ # if isinstance(scale, numbers.Integral):
+ # raise ValueError('scale cannot be an integer: %s' % scale)
+ #
+ # if isinstance(scale, numbers.Real):
+ # if scale < 0.:
+ # raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ # if scale >= 1.:
+ # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' % scale)
+ # if scale == 0.:
+ # logging.info('Scale of 0 disables regularizer.')
+ # return lambda _, name=None: None
+ #
+ # def lo(weights, name='lo_regularizer'):
+ # """Applies group column regularization to weights."""
+ # with tf.name_scope(name) as scope:
+ # my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ # # if tf.__version__ <= '0.12':
+ # # standard_ops_fn = standard_ops.mul
+ # # else:
+ # standard_ops_fn = standard_ops.multiply
+ # return standard_ops_fn(
+ # my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))),
+ # name=scope
+ # )
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def maxnorm_regularizer(scale=1.0):
+ """Max-norm regularization returns a function that can be used to apply max-norm regularization to weights.
+
+ More about max-norm, see `wiki-max norm `_.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ --------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ # if isinstance(scale, numbers.Integral):
+ # raise ValueError('scale cannot be an integer: %s' % scale)
+ #
+ # if isinstance(scale, numbers.Real):
+ # if scale < 0.:
+ # raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ # # if scale >= 1.:
+ # # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' %
+ # # scale)
+ # if scale == 0.:
+ # logging.info('Scale of 0 disables regularizer.')
+ # return lambda _, name=None: None
+ #
+ # def mn(weights, name='max_regularizer'):
+ # """Applies max-norm regularization to weights."""
+ # with tf.name_scope(name) as scope:
+ # my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ # # if tf.__version__ <= '0.12':
+ # # standard_ops_fn = standard_ops.mul
+ # # else:
+ # standard_ops_fn = standard_ops.multiply
+ # return standard_ops_fn(my_scale, standard_ops.reduce_max(standard_ops.abs(weights)), name=scope)
+ #
+ # return mn
+ raise NotImplementedError("Not Implemented.")
+
+
+def maxnorm_o_regularizer(scale):
+ """Max-norm output regularization removes the neurons of current layer.
+ Returns a function that can be used to apply max-norm regularization to each column of weight matrix.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn_o(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ---------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ # if isinstance(scale, numbers.Integral):
+ # raise ValueError('scale cannot be an integer: %s' % scale)
+ #
+ # if isinstance(scale, numbers.Real):
+ # if scale < 0.:
+ # raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ # # if scale >= 1.:
+ # # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' %
+ # # scale)
+ # if scale == 0.:
+ # logging.info('Scale of 0 disables regularizer.')
+ # return lambda _, name=None: None
+ #
+ # def mn_o(weights, name='maxnorm_o_regularizer'):
+ # """Applies max-norm regularization to weights."""
+ # with tf.name_scope(name) as scope:
+ # my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ # if tf.__version__ <= '0.12':
+ # standard_ops_fn = standard_ops.mul
+ # else:
+ # standard_ops_fn = standard_ops.multiply
+ # return standard_ops_fn(
+ # my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 0)), name=scope
+ # )
+ #
+ # return mn_o
+ raise NotImplementedError("Not Implemented.")
+
+
+def maxnorm_i_regularizer(scale):
+ """Max-norm input regularization removes the neurons of previous layer.
+ Returns a function that can be used to apply max-norm regularization to each row of weight matrix.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn_i(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ---------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ # if isinstance(scale, numbers.Integral):
+ # raise ValueError('scale cannot be an integer: %s' % scale)
+ #
+ # if isinstance(scale, numbers.Real):
+ # if scale < 0.:
+ # raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ # # if scale >= 1.:
+ # # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' %
+ # # scale)
+ # if scale == 0.:
+ # logging.info('Scale of 0 disables regularizer.')
+ # return lambda _, name=None: None
+ #
+ # def mn_i(weights, name='maxnorm_i_regularizer'):
+ # """Applies max-norm regularization to weights."""
+ # with tf.name_scope(name) as scope:
+ # my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ # if tf.__version__ <= '0.12':
+ # standard_ops_fn = standard_ops.mul
+ # else:
+ # standard_ops_fn = standard_ops.multiply
+ # return standard_ops_fn(
+ # my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope
+ # )
+ #
+ # return mn_i
+ raise NotImplementedError("Not Implemented.")
+
+
+def huber_loss(
+ output, target, is_mean=True, delta=1.0, dynamichuber=False, reverse=False, axis=-1, epsilon=0.00001, name=None
+):
+ """Huber Loss operation, see ``https://en.wikipedia.org/wiki/Huber_loss`` .
+ Reverse Huber Loss operation, see ''https://statweb.stanford.edu/~owen/reports/hhu.pdf''.
+ Dynamic Reverse Huber Loss operation, see ''https://arxiv.org/pdf/1606.00373.pdf''.
+
+ Parameters
+ ----------
+ output : Tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data (default).
+ - If False, use ``tf.reduce_sum``.
+ delta: float
+ The point where the huber loss function changes from a quadratic to linear.
+ dynamichuber: boolean
+ Whether compute the coefficient c for each batch.
+ - If True, c is 20% of the maximal per-batch error.
+ - If False, c is delta.
+ reverse: boolean
+ Whether compute the reverse huber loss.
+ axis : int or list of int
+ The dimensions to reduce.
+ epsilon:
+ Eplison.
+ name : string
+ Name of this loss.
+
+ """
+ # if reverse:
+ # if dynamichuber:
+ # huber_c = 0.2 * tf.reduce_max(tf.abs(output - target))
+ # else:
+ # huber_c = delta
+ # if is_mean:
+ # loss = tf.reduce_mean(
+ # tf.where(
+ # tf.less_equal(tf.abs(output - target), huber_c), tf.abs(output - target),
+ # tf.multiply(
+ # tf.pow(output - target, 2.0) + tf.pow(huber_c, 2.0),
+ # tf.math.divide_no_nan(.5, huber_c + epsilon)
+ # )
+ # ), name=name
+ # )
+ # else:
+ # loss = tf.reduce_mean(
+ # tf.reduce_sum(
+ # tf.where(
+ # tf.less_equal(tf.abs(output - target), huber_c), tf.abs(output - target),
+ # tf.multiply(
+ # tf.pow(output - target, 2.0) + tf.pow(huber_c, 2.0),
+ # tf.math.divide_no_nan(.5, huber_c + epsilon)
+ # )
+ # ), axis
+ # ), name=name
+ # )
+ # elif is_mean:
+ # loss = tf.reduce_mean(
+ # tf.where(
+ # tf.less_equal(tf.abs(output - target), delta), 0.5 * tf.pow(output - target, 2),
+ # delta * (tf.abs(output - target) - 0.5 * delta)
+ # ), name=name
+ # )
+ # else:
+ # loss = tf.reduce_mean(
+ # tf.reduce_sum(
+ # tf.where(
+ # tf.less_equal(tf.abs(output - target), delta), 0.5 * tf.pow(output - target, 2),
+ # delta * (tf.abs(output - target) - 0.5 * delta)
+ # ), axis
+ # ), name=name
+ # )
+ # return loss
+ raise NotImplementedError("Not Implemented.")
diff --git a/tensorlayer/cost/paddle_cost.py b/tensorlayer/cost/paddle_cost.py
new file mode 100644
index 0000000..d8a79f0
--- /dev/null
+++ b/tensorlayer/cost/paddle_cost.py
@@ -0,0 +1,548 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import paddle.nn.functional as F
+import paddle as pd
+
+__all__ = [
+ 'cross_entropy',
+ 'sigmoid_cross_entropy',
+ 'binary_cross_entropy',
+ 'mean_squared_error',
+ 'normalized_mean_square_error',
+ 'absolute_difference_error',
+ 'dice_coe',
+ 'dice_hard_coe',
+ 'iou_coe',
+ 'cross_entropy_seq',
+ 'cross_entropy_seq_with_mask',
+ 'cosine_similarity',
+ 'li_regularizer',
+ 'lo_regularizer',
+ 'maxnorm_regularizer',
+ 'maxnorm_o_regularizer',
+ 'maxnorm_i_regularizer',
+]
+
+def cross_entropy(output, target):
+ """Softmax cross-entropy operation, returns the TensorFlow expression of cross-entropy for two distributions,
+ it implements softmax internally. See ``tf.ops.sparse_softmax_cross_entropy_with_logits``.
+
+ Parameters
+ ----------
+ output : Tensor
+ A batch of distribution with shape: [batch_size, num of classes].
+ target : Tensor
+ A batch of index with shape: [batch_size, ].
+ name : string
+ Name of this loss.
+
+ Examples
+ --------
+ >>> import tensorlayer as tl
+ >>> ce = tl.cost.cross_entropy(y_logits, y_target_logits)
+
+ References
+ -----------
+ - About cross-entropy: ``__.
+ - The code is borrowed from: ``__.
+
+ """
+
+ return F.cross_entropy(input=output, label=target)
+
+
+def sigmoid_cross_entropy(output, target):
+ """Sigmoid cross-entropy operation, see ``tf.ops.sigmoid_cross_entropy_with_logits``.
+
+ Parameters
+ ----------
+ output : Tensor
+ A batch of distribution with shape: [batch_size, num of classes].
+ target : Tensor
+ A batch of index with shape: [batch_size, ].
+ name : string
+ Name of this loss.
+
+ """
+ # tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=target, logits=output), name=name)
+ depth = output.shape[-1]
+ label = pd.fluid.layers.one_hot(target, depth=depth)
+ out = pd.fluid.layers.sigmoid_cross_entropy_with_logits(x=output, label=label)
+ out = pd.fluid.layers.reduce_mean(out)
+ return out
+
+
+def binary_cross_entropy(output, target, epsilon=1e-8):
+ """Binary cross entropy operation.
+
+ Parameters
+ ----------
+ output : Tensor
+ Tensor with type of `float32` or `float64`.
+ target : Tensor
+ The target distribution, format the same with `output`.
+ epsilon : float
+ A small value to avoid output to be zero.
+ name : str
+ An optional name to attach to this function.
+
+ References
+ -----------
+ - `ericjang-DRAW `__
+
+ """
+ depth = output.shape[-1]
+ target = pd.fluid.layers.one_hot(target, depth=depth)
+ out = pd.fluid.layers.reduce_sum(
+ -(target * pd.log(output + epsilon) + (1. - target) * pd.log(1. - output + epsilon))
+ )
+ return out
+
+
+def mean_squared_error(output, target, is_mean=False, axis=-1, name="mean_squared_error"):
+ """Return the TensorFlow expression of mean-square-error (L2) of two batch of data.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data.
+ - If False, use ``tf.reduce_sum`` (default).
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ References
+ ------------
+ - `Wiki Mean Squared Error `__
+
+ """
+ depth = output.shape[-1]
+ target = pd.fluid.layers.one_hot(target, depth=depth)
+
+ if is_mean:
+ mse = F.mse_loss(input=output, label=target, reduction='mean')
+ else:
+ mse = F.mse_loss(input=output, label=target, reduction='sum')
+ return mse
+
+
+def normalized_mean_square_error(output, target, axis=-1, name="normalized_mean_squared_error_loss"):
+ """Return the TensorFlow expression of normalized mean-square-error of two distributions.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def absolute_difference_error(output, target, is_mean=False, axis=-1, name="absolute_difference_error_loss"):
+ """Return the TensorFlow expression of absolute difference error (L1) of two batch of data.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data.
+ - If False, use ``tf.reduce_sum`` (default).
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def dice_coe(output, target, loss_type='jaccard', axis=(1, 2, 3), smooth=1e-5):
+ """Soft dice (Sørensen or Jaccard) coefficient for comparing the similarity
+ of two batch of data, usually be used for binary image segmentation
+ i.e. labels are binary. The coefficient between 0 to 1, 1 means totally match.
+
+ Parameters
+ -----------
+ output : Tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : Tensor
+ The target distribution, format the same with `output`.
+ loss_type : str
+ ``jaccard`` or ``sorensen``, default is ``jaccard``.
+ axis : tuple of int
+ All dimensions are reduced, default ``[1,2,3]``.
+ smooth : float
+ This small value will be added to the numerator and denominator.
+ - If both output and target are empty, it makes sure dice is 1.
+ - If either output or target are empty (all pixels are background), dice = ```smooth/(small_value + smooth)``, then if smooth is very small, dice close to 0 (even the image values lower than the threshold), so in this case, higher smooth can have a higher dice.
+
+ Examples
+ ---------
+ >>> import tensorlayer as tl
+ >>> outputs = tl.act.pixel_wise_softmax(outputs)
+ >>> dice_loss = 1 - tl.cost.dice_coe(outputs, y_)
+
+ References
+ -----------
+ - `Wiki-Dice `__
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def dice_hard_coe(output, target, threshold=0.5, axis=(1, 2, 3), smooth=1e-5):
+ """Non-differentiable Sørensen–Dice coefficient for comparing the similarity
+ of two batch of data, usually be used for binary image segmentation i.e. labels are binary.
+ The coefficient between 0 to 1, 1 if totally match.
+
+ Parameters
+ -----------
+ output : tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : tensor
+ The target distribution, format the same with `output`.
+ threshold : float
+ The threshold value to be true.
+ axis : tuple of integer
+ All dimensions are reduced, default ``(1,2,3)``.
+ smooth : float
+ This small value will be added to the numerator and denominator, see ``dice_coe``.
+
+ References
+ -----------
+ - `Wiki-Dice `__
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def iou_coe(output, target, threshold=0.5, axis=(1, 2, 3), smooth=1e-5):
+ """Non-differentiable Intersection over Union (IoU) for comparing the
+ similarity of two batch of data, usually be used for evaluating binary image segmentation.
+ The coefficient between 0 to 1, and 1 means totally match.
+
+ Parameters
+ -----------
+ output : tensor
+ A batch of distribution with shape: [batch_size, ....], (any dimensions).
+ target : tensor
+ The target distribution, format the same with `output`.
+ threshold : float
+ The threshold value to be true.
+ axis : tuple of integer
+ All dimensions are reduced, default ``(1,2,3)``.
+ smooth : float
+ This small value will be added to the numerator and denominator, see ``dice_coe``.
+
+ Notes
+ ------
+ - IoU cannot be used as training loss, people usually use dice coefficient for training, IoU and hard-dice for evaluating.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def sequence_loss_by_example(
+ logits, targets, weights, average_across_timesteps=True, softmax_loss_function=None, name=None
+):
+ """Weighted cross-entropy loss for a sequence of logits (per example). see original tensorflow code :
+
+
+ Parameters
+ ----------
+ logits: List
+ List of 2D Tensors of shape [batch_size x num_decoder_symbols].
+ targets: List
+ List of 1D batch-sized int32 Tensors of the same length as logits.
+ weights: List
+ List of 1D batch-sized float-Tensors of the same length as logits.
+ average_across_timesteps: Boolean
+ If set, divide the returned cost by the total label weight.
+ softmax_loss_function: None or Function
+ Function (labels, logits) -> loss-batch to be used instead of the standard softmax (the default if this is None).
+ **Note that to avoid confusion, it is required for the function to accept named arguments.**
+ name: None or str
+ Optional name for this operation, default: "sequence_loss_by_example".
+
+ Returns
+ -------
+ 1D batch-sized float Tensor: The log-perplexity for each sequence.
+
+ Raises
+ ------
+ ValueError: If len(logits) is different from len(targets) or len(weights).
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def cross_entropy_seq(logits, target_seqs, batch_size=None):
+ """Returns the expression of cross-entropy of two sequences, implement
+ softmax internally. Normally be used for fixed length RNN outputs, see `PTB example `__.
+
+ Parameters
+ ----------
+ logits : Tensor
+ 2D tensor with shape of `[batch_size * n_steps, n_classes]`.
+ target_seqs : Tensor
+ The target sequence, 2D tensor `[batch_size, n_steps]`, if the number of step is dynamic, please use ``tl.cost.cross_entropy_seq_with_mask`` instead.
+ batch_size : None or int.
+ Whether to divide the cost by batch size.
+ - If integer, the return cost will be divided by `batch_size`.
+ - If None (default), the return cost will not be divided by anything.
+
+ Examples
+ --------
+ >>> import tensorlayer as tl
+ >>> # see `PTB example `__.for more details
+ >>> # outputs shape : (batch_size * n_steps, n_classes)
+ >>> # targets shape : (batch_size, n_steps)
+ >>> cost = tl.cost.cross_entropy_seq(outputs, targets)
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def cross_entropy_seq_with_mask(logits, target_seqs, input_mask, return_details=False, name=None):
+ """Returns the expression of cross-entropy of two sequences, implement
+ softmax internally. Normally be used for Dynamic RNN with Synced sequence input and output.
+
+ Parameters
+ -----------
+ logits : Tensor
+ 2D tensor with shape of [batch_size * ?, n_classes], `?` means dynamic IDs for each example.
+ - Can be get from `DynamicRNNLayer` by setting ``return_seq_2d`` to `True`.
+ target_seqs : Tensor
+ int of tensor, like word ID. [batch_size, ?], `?` means dynamic IDs for each example.
+ input_mask : Tensor
+ The mask to compute loss, it has the same size with `target_seqs`, normally 0 or 1.
+ return_details : boolean
+ Whether to return detailed losses.
+ - If False (default), only returns the loss.
+ - If True, returns the loss, losses, weights and targets (see source code).
+
+ Examples
+ --------
+ >>> import tensorlayer as tl
+ >>> import tensorflow as tf
+ >>> import numpy as np
+ >>> batch_size = 64
+ >>> vocab_size = 10000
+ >>> embedding_size = 256
+ >>> ni = tl.layers.Input([batch_size, None], dtype=tf.int64)
+ >>> net = tl.layers.Embedding(
+ ... vocabulary_size = vocab_size,
+ ... embedding_size = embedding_size,
+ ... name = 'seq_embedding')(ni)
+ >>> net = tl.layers.RNN(
+ ... cell =tf.keras.layers.LSTMCell(units=embedding_size, dropout=0.1),
+ ... return_seq_2d = True,
+ ... name = 'dynamicrnn')(net)
+ >>> net = tl.layers.Dense(n_units=vocab_size, name="output")(net)
+ >>> model = tl.models.Model(inputs=ni, outputs=net)
+ >>> input_seqs = np.random.randint(0, 10, size=(batch_size, 10), dtype=np.int64)
+ >>> target_seqs = np.random.randint(0, 10, size=(batch_size, 10), dtype=np.int64)
+ >>> input_mask = np.random.randint(0, 2, size=(batch_size, 10), dtype=np.int64)
+ >>> outputs = model(input_seqs, is_train=True)
+ >>> loss = tl.cost.cross_entropy_seq_with_mask(outputs, target_seqs, input_mask)
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def cosine_similarity(v1, v2):
+ """Cosine similarity [-1, 1].
+
+ Parameters
+ ----------
+ v1, v2 : Tensor
+ Tensor with the same shape [batch_size, n_feature].
+
+ References
+ ----------
+ - `Wiki `__.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+# Regularization Functions
+def li_regularizer(scale, scope=None):
+ """Li regularization removes the neurons of previous layer. The `i` represents `inputs`.
+ Returns a function that can be used to apply group li regularization to weights.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+ scope: str
+ An optional scope name for this function.
+
+ Returns
+ --------
+ A function with signature `li(weights, name=None)` that apply Li regularization.
+
+ Raises
+ ------
+ ValueError : if scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def lo_regularizer(scale):
+ """Lo regularization removes the neurons of current layer. The `o` represents `outputs`
+ Returns a function that can be used to apply group lo regularization to weights.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ -------
+ A function with signature `lo(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def maxnorm_regularizer(scale=1.0):
+ """Max-norm regularization returns a function that can be used to apply max-norm regularization to weights.
+
+ More about max-norm, see `wiki-max norm `_.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ --------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def maxnorm_o_regularizer(scale):
+ """Max-norm output regularization removes the neurons of current layer.
+ Returns a function that can be used to apply max-norm regularization to each column of weight matrix.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn_o(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ---------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def maxnorm_i_regularizer(scale):
+ """Max-norm input regularization removes the neurons of previous layer.
+ Returns a function that can be used to apply max-norm regularization to each row of weight matrix.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn_i(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ---------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
+
+
+def huber_loss(
+ output, target, is_mean=True, delta=1.0, dynamichuber=False, reverse=False, axis=-1, epsilon=0.00001, name=None
+):
+ """Huber Loss operation, see ``https://en.wikipedia.org/wiki/Huber_loss`` .
+ Reverse Huber Loss operation, see ''https://statweb.stanford.edu/~owen/reports/hhu.pdf''.
+ Dynamic Reverse Huber Loss operation, see ''https://arxiv.org/pdf/1606.00373.pdf''.
+
+ Parameters
+ ----------
+ output : Tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data (default).
+ - If False, use ``tf.reduce_sum``.
+ delta: float
+ The point where the huber loss function changes from a quadratic to linear.
+ dynamichuber: boolean
+ Whether compute the coefficient c for each batch.
+ - If True, c is 20% of the maximal per-batch error.
+ - If False, c is delta.
+ reverse: boolean
+ Whether compute the reverse huber loss.
+ axis : int or list of int
+ The dimensions to reduce.
+ epsilon:
+ Eplison.
+ name : string
+ Name of this loss.
+
+ """
+
+ raise NotImplementedError("Not Implemented.")
diff --git a/tensorlayer/cost/tensorflow_cost.py b/tensorlayer/cost/tensorflow_cost.py
new file mode 100644
index 0000000..b07acad
--- /dev/null
+++ b/tensorlayer/cost/tensorflow_cost.py
@@ -0,0 +1,860 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import numbers
+
+import tensorflow as tf
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops, math_ops, nn_ops, standard_ops
+
+from tensorlayer import logging
+
+__all__ = [
+ 'cross_entropy',
+ 'sigmoid_cross_entropy',
+ 'binary_cross_entropy',
+ 'mean_squared_error',
+ 'normalized_mean_square_error',
+ 'absolute_difference_error',
+ 'dice_coe',
+ 'dice_hard_coe',
+ 'iou_coe',
+ 'cross_entropy_seq',
+ 'cross_entropy_seq_with_mask',
+ 'cosine_similarity',
+ 'li_regularizer',
+ 'lo_regularizer',
+ 'maxnorm_regularizer',
+ 'maxnorm_o_regularizer',
+ 'maxnorm_i_regularizer',
+]
+
+
+def cross_entropy(output, target, name=None):
+ """Softmax cross-entropy operation, returns the TensorFlow expression of cross-entropy for two distributions,
+ it implements softmax internally. See ``tf.ops.sparse_softmax_cross_entropy_with_logits``.
+
+ Parameters
+ ----------
+ output : Tensor
+ A batch of distribution with shape: [batch_size, num of classes].
+ target : Tensor
+ A batch of index with shape: [batch_size, ].
+ name : string
+ Name of this loss.
+
+ Examples
+ --------
+ >>> import tensorlayer as tl
+ >>> ce = tl.cost.cross_entropy(y_logits, y_target_logits, 'my_loss')
+
+ References
+ -----------
+ - About cross-entropy: ``__.
+ - The code is borrowed from: ``__.
+
+ """
+ # if name is None:
+ # raise Exception("Please give a unique name to tl.cost.cross_entropy for TF1.0+")
+ return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=output), name=name)
+
+
+def sigmoid_cross_entropy(output, target, name=None):
+ """Sigmoid cross-entropy operation, see ``tf.ops.sigmoid_cross_entropy_with_logits``.
+
+ Parameters
+ ----------
+ output : Tensor
+ A batch of distribution with shape: [batch_size, num of classes].
+ target : Tensor
+ A batch of index with shape: [batch_size, ].
+ name : string
+ Name of this loss.
+
+ """
+ return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=target, logits=output), name=name)
+
+
+def binary_cross_entropy(output, target, epsilon=1e-8, name='bce_loss'):
+ """Binary cross entropy operation.
+
+ Parameters
+ ----------
+ output : Tensor
+ Tensor with type of `float32` or `float64`.
+ target : Tensor
+ The target distribution, format the same with `output`.
+ epsilon : float
+ A small value to avoid output to be zero.
+ name : str
+ An optional name to attach to this function.
+
+ References
+ -----------
+ - `ericjang-DRAW `__
+
+ """
+ # with ops.op_scope([output, target], name, "bce_loss") as name:
+ # output = ops.convert_to_tensor(output, name="preds")
+ # target = ops.convert_to_tensor(targets, name="target")
+
+ # with tf.name_scope(name):
+ return tf.reduce_mean(
+ tf.reduce_sum(
+ -(target * tf.math.log(output + epsilon) + (1. - target) * tf.math.log(1. - output + epsilon)), axis=1
+ ), name=name
+ )
+
+ # For brevity, let `x = output`, `z = target`. The binary cross entropy loss is
+ #
+ # loss(x, z) = - sum_i (x[i] * log(z[i]) + (1 - x[i]) * log(1 - z[i]))
+
+
+def mean_squared_error(output, target, is_mean=False, axis=-1, name="mean_squared_error"):
+ """Return the TensorFlow expression of mean-square-error (L2) of two batch of data.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data.
+ - If False, use ``tf.reduce_sum`` (default).
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ References
+ ------------
+ - `Wiki Mean Squared Error `__
+
+ """
+ # with tf.name_scope(name):
+ # if len(output.shape) == 2: # [batch_size, n_feature]
+ # axis = 1
+ # elif len(output.shape) == 3: # [batch_size, w, h]
+ # axis = [1, 2]
+ # elif len(output.shape) == 4: # [batch_size, w, h, c]
+ # axis = [1, 2, 3]
+ # else:
+ # raise Exception("Unknow dimension")
+
+ if is_mean:
+ mse = tf.reduce_mean(tf.reduce_mean(tf.math.squared_difference(output, target), axis), name=name)
+ else:
+ mse = tf.reduce_mean(tf.reduce_sum(tf.math.squared_difference(output, target), axis), name=name)
+ return mse
+
+
+def normalized_mean_square_error(output, target, axis=-1, name="normalized_mean_squared_error_loss"):
+ """Return the TensorFlow expression of normalized mean-square-error of two distributions.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ """
+ with tf.name_scope("normalized_mean_squared_error_loss"):
+ # if len(output.shape) == 2: # [batch_size, n_feature]
+ # axis = 1
+ # elif len(output.shape) == 3: # [batch_size, w, h]
+ # axis = [1, 2]
+ # elif len(output.shape) == 4: # [batch_size, w, h, c]
+ # axis = [1, 2, 3]
+ nmse_a = tf.sqrt(tf.reduce_sum(tf.math.squared_difference(output, target), axis=axis))
+ nmse_b = tf.sqrt(tf.reduce_sum(tf.square(target), axis=axis))
+ nmse = tf.reduce_mean(nmse_a / nmse_b, name=name)
+ return nmse
+
+
+def absolute_difference_error(output, target, is_mean=False, axis=-1, name="absolute_difference_error_loss"):
+ """Return the TensorFlow expression of absolute difference error (L1) of two batch of data.
+
+ Parameters
+ ----------
+ output : Tensor
+ 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data.
+ - If False, use ``tf.reduce_sum`` (default).
+ axis : int or list of int
+ The dimensions to reduce.
+ name : str
+ An optional name to attach to this function.
+
+ """
+ # # with tf.name_scope("absolute_difference_error_loss"):
+ # if len(output.shape) == 2: # [batch_size, n_feature]
+ # axis = 1
+ # elif len(output.shape) == 3: # [batch_size, w, h]
+ # axis = [1, 2]
+ # elif len(output.shape) == 4: # [batch_size, w, h, c]
+ # axis = [1, 2, 3]
+ # else:
+ # raise Exception("Unknow dimension")
+ if is_mean:
+ loss = tf.reduce_mean(tf.reduce_mean(tf.abs(output - target), axis), name=name)
+ else:
+ loss = tf.reduce_mean(tf.reduce_sum(tf.abs(output - target), axis), name=name)
+ return loss
+
+
+def dice_coe(output, target, loss_type='jaccard', axis=(1, 2, 3), smooth=1e-5):
+ """Soft dice (Sørensen or Jaccard) coefficient for comparing the similarity
+ of two batch of data, usually be used for binary image segmentation
+ i.e. labels are binary. The coefficient between 0 to 1, 1 means totally match.
+
+ Parameters
+ -----------
+ output : Tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : Tensor
+ The target distribution, format the same with `output`.
+ loss_type : str
+ ``jaccard`` or ``sorensen``, default is ``jaccard``.
+ axis : tuple of int
+ All dimensions are reduced, default ``[1,2,3]``.
+ smooth : float
+ This small value will be added to the numerator and denominator.
+ - If both output and target are empty, it makes sure dice is 1.
+ - If either output or target are empty (all pixels are background), dice = ```smooth/(small_value + smooth)``, then if smooth is very small, dice close to 0 (even the image values lower than the threshold), so in this case, higher smooth can have a higher dice.
+
+ Examples
+ ---------
+ >>> import tensorlayer as tl
+ >>> outputs = tl.act.pixel_wise_softmax(outputs)
+ >>> dice_loss = 1 - tl.cost.dice_coe(outputs, y_)
+
+ References
+ -----------
+ - `Wiki-Dice `__
+
+ """
+ inse = tf.reduce_sum(output * target, axis=axis)
+ if loss_type == 'jaccard':
+ l = tf.reduce_sum(output * output, axis=axis)
+ r = tf.reduce_sum(target * target, axis=axis)
+ elif loss_type == 'sorensen':
+ l = tf.reduce_sum(output, axis=axis)
+ r = tf.reduce_sum(target, axis=axis)
+ else:
+ raise Exception("Unknow loss_type")
+ # old axis=[0,1,2,3]
+ # dice = 2 * (inse) / (l + r)
+ # epsilon = 1e-5
+ # dice = tf.clip_by_value(dice, 0, 1.0-epsilon) # if all empty, dice = 1
+ # new haodong
+ dice = (2. * inse + smooth) / (l + r + smooth)
+ ##
+ dice = tf.reduce_mean(dice, name='dice_coe')
+ return dice
+
+
+def dice_hard_coe(output, target, threshold=0.5, axis=(1, 2, 3), smooth=1e-5):
+ """Non-differentiable Sørensen–Dice coefficient for comparing the similarity
+ of two batch of data, usually be used for binary image segmentation i.e. labels are binary.
+ The coefficient between 0 to 1, 1 if totally match.
+
+ Parameters
+ -----------
+ output : tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : tensor
+ The target distribution, format the same with `output`.
+ threshold : float
+ The threshold value to be true.
+ axis : tuple of integer
+ All dimensions are reduced, default ``(1,2,3)``.
+ smooth : float
+ This small value will be added to the numerator and denominator, see ``dice_coe``.
+
+ References
+ -----------
+ - `Wiki-Dice `__
+
+ """
+ output = tf.cast(output > threshold, dtype=tf.float32)
+ target = tf.cast(target > threshold, dtype=tf.float32)
+ inse = tf.reduce_sum(tf.multiply(output, target), axis=axis)
+ l = tf.reduce_sum(output, axis=axis)
+ r = tf.reduce_sum(target, axis=axis)
+ # old axis=[0,1,2,3]
+ # hard_dice = 2 * (inse) / (l + r)
+ # epsilon = 1e-5
+ # hard_dice = tf.clip_by_value(hard_dice, 0, 1.0-epsilon)
+ # new haodong
+ hard_dice = (2. * inse + smooth) / (l + r + smooth)
+ ##
+ hard_dice = tf.reduce_mean(hard_dice, name='hard_dice')
+ return hard_dice
+
+
+def iou_coe(output, target, threshold=0.5, axis=(1, 2, 3), smooth=1e-5):
+ """Non-differentiable Intersection over Union (IoU) for comparing the
+ similarity of two batch of data, usually be used for evaluating binary image segmentation.
+ The coefficient between 0 to 1, and 1 means totally match.
+
+ Parameters
+ -----------
+ output : tensor
+ A batch of distribution with shape: [batch_size, ....], (any dimensions).
+ target : tensor
+ The target distribution, format the same with `output`.
+ threshold : float
+ The threshold value to be true.
+ axis : tuple of integer
+ All dimensions are reduced, default ``(1,2,3)``.
+ smooth : float
+ This small value will be added to the numerator and denominator, see ``dice_coe``.
+
+ Notes
+ ------
+ - IoU cannot be used as training loss, people usually use dice coefficient for training, IoU and hard-dice for evaluating.
+
+ """
+ pre = tf.cast(output > threshold, dtype=tf.float32)
+ truth = tf.cast(target > threshold, dtype=tf.float32)
+ inse = tf.reduce_sum(tf.multiply(pre, truth), axis=axis) # AND
+ union = tf.reduce_sum(tf.cast(tf.add(pre, truth) >= 1, dtype=tf.float32), axis=axis) # OR
+ # old axis=[0,1,2,3]
+ # epsilon = 1e-5
+ # batch_iou = inse / (union + epsilon)
+ # new haodong
+ batch_iou = (inse + smooth) / (union + smooth)
+ iou = tf.reduce_mean(batch_iou, name='iou_coe')
+ return iou # , pre, truth, inse, union
+
+
+# ## test soft/hard dice and iou
+# import numpy as np
+# y = np.zeros((1,10,10,1))
+# # y[0,0:5,0:5]=1.0
+# o = np.zeros((1,10,10,1))
+# # o[:,:,:,:] = 0 # what we want: dice=0 iou=0 OK
+# # o[0,0:2,0:2]=0.3 # what we want: dice larger iou=0 OK
+# # o[0,0:2,0:2]=0.6 # what we want: dice larger iou small OK
+# # o[0,0:3,0:3]=0.6 # what we want: dice larger iou larger OK
+# # o[0,0:3,0:3]=1 # what we want: dice larger iou same OK
+# # o[0,0:5,0:5]=1 # what we want: dice=1 iou=1 OK
+# # o[0,0:5,0:5]=0.3 # what we want: dice smaller iou=0 OK
+# # o[0,0:5,0:5]=1e-2 # what we want: dice≈0 iou=0 OK
+# # o[0,8:10,8:10]=1.0 # what we want: dice=0 iou=0 OK
+# # o[0,8:10,8:10]=1e-10 # what we want: dice=0 iou=0 OK
+# # y[:,:,:,:] = o[:,:,:,:] = 0 # what we want: dice=1 iou=1 OK
+# ## why in u-net, dice=1 hard-dice=1 iou=1 exist?? print bug?
+#
+# d = dice_coe(o, y, 'jaccard', smooth=1.)
+# hd = dice_hard_coe(o, y, smooth=1e-5)
+# i = iou_coe(o, y, smooth=1e-5)
+# sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
+# # sess.run(tf.local_variables_initializer())
+# print(sess.run([d,hd,i]))
+# # p, t, i, u = sess.run([pre, truth, inse, union])
+# # import pprint
+# # pprint.pprint(((y>0.5)*(o>0.5)).astype(int).tolist())
+# # pprint.pprint(p.tolist())
+# # pprint.pprint(t.tolist())
+# # pprint.pprint(i)
+# # pprint.pprint(u)
+# exit()
+
+
+def sequence_loss_by_example(
+ logits, targets, weights, average_across_timesteps=True, softmax_loss_function=None, name=None
+):
+ """Weighted cross-entropy loss for a sequence of logits (per example). see original tensorflow code :
+
+
+ Parameters
+ ----------
+ logits: List
+ List of 2D Tensors of shape [batch_size x num_decoder_symbols].
+ targets: List
+ List of 1D batch-sized int32 Tensors of the same length as logits.
+ weights: List
+ List of 1D batch-sized float-Tensors of the same length as logits.
+ average_across_timesteps: Boolean
+ If set, divide the returned cost by the total label weight.
+ softmax_loss_function: None or Function
+ Function (labels, logits) -> loss-batch to be used instead of the standard softmax (the default if this is None).
+ **Note that to avoid confusion, it is required for the function to accept named arguments.**
+ name: None or str
+ Optional name for this operation, default: "sequence_loss_by_example".
+
+ Returns
+ -------
+ 1D batch-sized float Tensor: The log-perplexity for each sequence.
+
+ Raises
+ ------
+ ValueError: If len(logits) is different from len(targets) or len(weights).
+
+ """
+ if len(targets) != len(logits) or len(weights) != len(logits):
+ raise ValueError(
+ "Lengths of logits, weights, and targets must be the same "
+ "%d, %d, %d." % (len(logits), len(weights), len(targets))
+ )
+ with ops.name_scope(name, "sequence_loss_by_example", logits + targets + weights):
+ log_perp_list = []
+ for logit, target, weight in zip(logits, targets, weights):
+ if softmax_loss_function is None:
+ # TODO(irving,ebrevdo): This reshape is needed because
+ # sequence_loss_by_example is called with scalars sometimes, which
+ # violates our general scalar strictness policy.
+ target = array_ops.reshape(target, [-1])
+ crossent = nn_ops.sparse_softmax_cross_entropy_with_logits(labels=target, logits=logit)
+ else:
+ crossent = softmax_loss_function(labels=target, logits=logit)
+ log_perp_list.append(crossent * weight)
+ log_perps = math_ops.add_n(log_perp_list)
+ if average_across_timesteps:
+ total_size = math_ops.add_n(weights)
+ total_size += 1e-12 # Just to avoid division by 0 for all-0 weights.
+ log_perps /= total_size
+ return log_perps
+
+
+def cross_entropy_seq(logits, target_seqs, batch_size=None):
+ """Returns the expression of cross-entropy of two sequences, implement
+ softmax internally. Normally be used for fixed length RNN outputs, see `PTB example `__.
+
+ Parameters
+ ----------
+ logits : Tensor
+ 2D tensor with shape of `[batch_size * n_steps, n_classes]`.
+ target_seqs : Tensor
+ The target sequence, 2D tensor `[batch_size, n_steps]`, if the number of step is dynamic, please use ``tl.cost.cross_entropy_seq_with_mask`` instead.
+ batch_size : None or int.
+ Whether to divide the cost by batch size.
+ - If integer, the return cost will be divided by `batch_size`.
+ - If None (default), the return cost will not be divided by anything.
+
+ Examples
+ --------
+ >>> import tensorlayer as tl
+ >>> # see `PTB example `__.for more details
+ >>> # outputs shape : (batch_size * n_steps, n_classes)
+ >>> # targets shape : (batch_size, n_steps)
+ >>> cost = tl.cost.cross_entropy_seq(outputs, targets)
+
+ """
+ sequence_loss_by_example_fn = sequence_loss_by_example
+
+ loss = sequence_loss_by_example_fn(
+ [logits], [tf.reshape(target_seqs, [-1])], [tf.ones_like(tf.reshape(target_seqs, [-1]), dtype=tf.float32)]
+ )
+ # [tf.ones([batch_size * num_steps])])
+ cost = tf.reduce_sum(loss) # / batch_size
+ if batch_size is not None:
+ cost = cost / batch_size
+ return cost
+
+
+def cross_entropy_seq_with_mask(logits, target_seqs, input_mask, return_details=False, name=None):
+ """Returns the expression of cross-entropy of two sequences, implement
+ softmax internally. Normally be used for Dynamic RNN with Synced sequence input and output.
+
+ Parameters
+ -----------
+ logits : Tensor
+ 2D tensor with shape of [batch_size * ?, n_classes], `?` means dynamic IDs for each example.
+ - Can be get from `DynamicRNNLayer` by setting ``return_seq_2d`` to `True`.
+ target_seqs : Tensor
+ int of tensor, like word ID. [batch_size, ?], `?` means dynamic IDs for each example.
+ input_mask : Tensor
+ The mask to compute loss, it has the same size with `target_seqs`, normally 0 or 1.
+ return_details : boolean
+ Whether to return detailed losses.
+ - If False (default), only returns the loss.
+ - If True, returns the loss, losses, weights and targets (see source code).
+
+ Examples
+ --------
+ >>> import tensorlayer as tl
+ >>> import tensorflow as tf
+ >>> import numpy as np
+ >>> batch_size = 64
+ >>> vocab_size = 10000
+ >>> embedding_size = 256
+ >>> ni = tl.layers.Input([batch_size, None], dtype=tf.int64)
+ >>> net = tl.layers.Embedding(
+ ... vocabulary_size = vocab_size,
+ ... embedding_size = embedding_size,
+ ... name = 'seq_embedding')(ni)
+ >>> net = tl.layers.RNN(
+ ... cell =tf.keras.layers.LSTMCell(units=embedding_size, dropout=0.1),
+ ... return_seq_2d = True,
+ ... name = 'dynamicrnn')(net)
+ >>> net = tl.layers.Dense(n_units=vocab_size, name="output")(net)
+ >>> model = tl.models.Model(inputs=ni, outputs=net)
+ >>> input_seqs = np.random.randint(0, 10, size=(batch_size, 10), dtype=np.int64)
+ >>> target_seqs = np.random.randint(0, 10, size=(batch_size, 10), dtype=np.int64)
+ >>> input_mask = np.random.randint(0, 2, size=(batch_size, 10), dtype=np.int64)
+ >>> outputs = model(input_seqs, is_train=True)
+ >>> loss = tl.cost.cross_entropy_seq_with_mask(outputs, target_seqs, input_mask)
+
+ """
+ targets = tf.reshape(target_seqs, [-1]) # to one vector
+ weights = tf.cast(tf.reshape(input_mask, [-1]), dtype=tf.float32) # to one vector like targets
+ losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets, name=name) * weights
+ # losses = tf.reduce_mean(tf.ops.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets, name=name)) # for TF1.0 and others
+
+ loss = tf.divide(
+ tf.reduce_sum(losses), # loss from mask. reduce_sum before element-wise mul with mask !!
+ tf.reduce_sum(weights),
+ name="seq_loss_with_mask"
+ )
+
+ if return_details:
+ return loss, losses, weights, targets
+ else:
+ return loss
+
+
+def cosine_similarity(v1, v2):
+ """Cosine similarity [-1, 1].
+
+ Parameters
+ ----------
+ v1, v2 : Tensor
+ Tensor with the same shape [batch_size, n_feature].
+
+ References
+ ----------
+ - `Wiki `__.
+
+ """
+
+ return tf.reduce_sum(tf.multiply(v1, v2), 1) / \
+ (tf.sqrt(tf.reduce_sum(tf.multiply(v1, v1), 1)) *
+ tf.sqrt(tf.reduce_sum(tf.multiply(v2, v2), 1)))
+
+
+# Regularization Functions
+def li_regularizer(scale, scope=None):
+ """Li regularization removes the neurons of previous layer. The `i` represents `inputs`.
+ Returns a function that can be used to apply group li regularization to weights.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+ scope: str
+ An optional scope name for this function.
+
+ Returns
+ --------
+ A function with signature `li(weights, name=None)` that apply Li regularization.
+
+ Raises
+ ------
+ ValueError : if scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ if isinstance(scale, numbers.Integral):
+ raise ValueError('scale cannot be an integer: %s' % scale)
+ if isinstance(scale, numbers.Real):
+ if scale < 0.:
+ raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ if scale >= 1.:
+ raise ValueError('Setting a scale greater than 1 on a regularizer: %g' % scale)
+ if scale == 0.:
+ logging.info('Scale of 0 disables regularizer.')
+ return lambda _, name=None: None
+
+ def li(weights):
+ """Applies li regularization to weights."""
+ with tf.name_scope('li_regularizer') as scope:
+ my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ # if tf.__version__ <= '0.12':
+ # standard_ops_fn = standard_ops.mul
+ # else:
+ standard_ops_fn = standard_ops.multiply
+ return standard_ops_fn(
+ my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))),
+ name=scope
+ )
+
+ return li
+
+
+def lo_regularizer(scale):
+ """Lo regularization removes the neurons of current layer. The `o` represents `outputs`
+ Returns a function that can be used to apply group lo regularization to weights.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ -------
+ A function with signature `lo(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ if isinstance(scale, numbers.Integral):
+ raise ValueError('scale cannot be an integer: %s' % scale)
+
+ if isinstance(scale, numbers.Real):
+ if scale < 0.:
+ raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ if scale >= 1.:
+ raise ValueError('Setting a scale greater than 1 on a regularizer: %g' % scale)
+ if scale == 0.:
+ logging.info('Scale of 0 disables regularizer.')
+ return lambda _, name=None: None
+
+ def lo(weights, name='lo_regularizer'):
+ """Applies group column regularization to weights."""
+ with tf.name_scope(name) as scope:
+ my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ # if tf.__version__ <= '0.12':
+ # standard_ops_fn = standard_ops.mul
+ # else:
+ standard_ops_fn = standard_ops.multiply
+ return standard_ops_fn(
+ my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))),
+ name=scope
+ )
+
+ return lo
+
+
+def maxnorm_regularizer(scale=1.0):
+ """Max-norm regularization returns a function that can be used to apply max-norm regularization to weights.
+
+ More about max-norm, see `wiki-max norm `_.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ --------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ if isinstance(scale, numbers.Integral):
+ raise ValueError('scale cannot be an integer: %s' % scale)
+
+ if isinstance(scale, numbers.Real):
+ if scale < 0.:
+ raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ # if scale >= 1.:
+ # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' %
+ # scale)
+ if scale == 0.:
+ logging.info('Scale of 0 disables regularizer.')
+ return lambda _, name=None: None
+
+ def mn(weights, name='max_regularizer'):
+ """Applies max-norm regularization to weights."""
+ with tf.name_scope(name) as scope:
+ my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ # if tf.__version__ <= '0.12':
+ # standard_ops_fn = standard_ops.mul
+ # else:
+ standard_ops_fn = standard_ops.multiply
+ return standard_ops_fn(my_scale, standard_ops.reduce_max(standard_ops.abs(weights)), name=scope)
+
+ return mn
+
+
+def maxnorm_o_regularizer(scale):
+ """Max-norm output regularization removes the neurons of current layer.
+ Returns a function that can be used to apply max-norm regularization to each column of weight matrix.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn_o(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ---------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ if isinstance(scale, numbers.Integral):
+ raise ValueError('scale cannot be an integer: %s' % scale)
+
+ if isinstance(scale, numbers.Real):
+ if scale < 0.:
+ raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ # if scale >= 1.:
+ # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' %
+ # scale)
+ if scale == 0.:
+ logging.info('Scale of 0 disables regularizer.')
+ return lambda _, name=None: None
+
+ def mn_o(weights, name='maxnorm_o_regularizer'):
+ """Applies max-norm regularization to weights."""
+ with tf.name_scope(name) as scope:
+ my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ if tf.__version__ <= '0.12':
+ standard_ops_fn = standard_ops.mul
+ else:
+ standard_ops_fn = standard_ops.multiply
+ return standard_ops_fn(
+ my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 0)), name=scope
+ )
+
+ return mn_o
+
+
+def maxnorm_i_regularizer(scale):
+ """Max-norm input regularization removes the neurons of previous layer.
+ Returns a function that can be used to apply max-norm regularization to each row of weight matrix.
+ The implementation follows `TensorFlow contrib `__.
+
+ Parameters
+ ----------
+ scale : float
+ A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+
+ Returns
+ ---------
+ A function with signature `mn_i(weights, name=None)` that apply Lo regularization.
+
+ Raises
+ ---------
+ ValueError : If scale is outside of the range [0.0, 1.0] or if scale is not a float.
+
+ """
+ if isinstance(scale, numbers.Integral):
+ raise ValueError('scale cannot be an integer: %s' % scale)
+
+ if isinstance(scale, numbers.Real):
+ if scale < 0.:
+ raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)
+ # if scale >= 1.:
+ # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' %
+ # scale)
+ if scale == 0.:
+ logging.info('Scale of 0 disables regularizer.')
+ return lambda _, name=None: None
+
+ def mn_i(weights, name='maxnorm_i_regularizer'):
+ """Applies max-norm regularization to weights."""
+ with tf.name_scope(name) as scope:
+ my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
+ if tf.__version__ <= '0.12':
+ standard_ops_fn = standard_ops.mul
+ else:
+ standard_ops_fn = standard_ops.multiply
+ return standard_ops_fn(
+ my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope
+ )
+
+ return mn_i
+
+
+def huber_loss(
+ output, target, is_mean=True, delta=1.0, dynamichuber=False, reverse=False, axis=-1, epsilon=0.00001, name=None
+):
+ """Huber Loss operation, see ``https://en.wikipedia.org/wiki/Huber_loss`` .
+ Reverse Huber Loss operation, see ''https://statweb.stanford.edu/~owen/reports/hhu.pdf''.
+ Dynamic Reverse Huber Loss operation, see ''https://arxiv.org/pdf/1606.00373.pdf''.
+
+ Parameters
+ ----------
+ output : Tensor
+ A distribution with shape: [batch_size, ....], (any dimensions).
+ target : Tensor
+ The target distribution, format the same with `output`.
+ is_mean : boolean
+ Whether compute the mean or sum for each example.
+ - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data (default).
+ - If False, use ``tf.reduce_sum``.
+ delta: float
+ The point where the huber loss function changes from a quadratic to linear.
+ dynamichuber: boolean
+ Whether compute the coefficient c for each batch.
+ - If True, c is 20% of the maximal per-batch error.
+ - If False, c is delta.
+ reverse: boolean
+ Whether compute the reverse huber loss.
+ axis : int or list of int
+ The dimensions to reduce.
+ epsilon:
+ Eplison.
+ name : string
+ Name of this loss.
+
+ """
+ if reverse:
+ if dynamichuber:
+ huber_c = 0.2 * tf.reduce_max(tf.abs(output - target))
+ else:
+ huber_c = delta
+ if is_mean:
+ loss = tf.reduce_mean(
+ tf.where(
+ tf.less_equal(tf.abs(output - target), huber_c), tf.abs(output - target),
+ tf.multiply(
+ tf.pow(output - target, 2.0) + tf.pow(huber_c, 2.0),
+ tf.math.divide_no_nan(.5, huber_c + epsilon)
+ )
+ ), name=name
+ )
+ else:
+ loss = tf.reduce_mean(
+ tf.reduce_sum(
+ tf.where(
+ tf.less_equal(tf.abs(output - target), huber_c), tf.abs(output - target),
+ tf.multiply(
+ tf.pow(output - target, 2.0) + tf.pow(huber_c, 2.0),
+ tf.math.divide_no_nan(.5, huber_c + epsilon)
+ )
+ ), axis
+ ), name=name
+ )
+ elif is_mean:
+ loss = tf.reduce_mean(
+ tf.where(
+ tf.less_equal(tf.abs(output - target), delta), 0.5 * tf.pow(output - target, 2),
+ delta * (tf.abs(output - target) - 0.5 * delta)
+ ), name=name
+ )
+ else:
+ loss = tf.reduce_mean(
+ tf.reduce_sum(
+ tf.where(
+ tf.less_equal(tf.abs(output - target), delta), 0.5 * tf.pow(output - target, 2),
+ delta * (tf.abs(output - target) - 0.5 * delta)
+ ), axis
+ ), name=name
+ )
+ return loss
diff --git a/tensorlayer/dataflow/__init__.py b/tensorlayer/dataflow/__init__.py
new file mode 100644
index 0000000..d26b322
--- /dev/null
+++ b/tensorlayer/dataflow/__init__.py
@@ -0,0 +1,23 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function
+
+from tensorlayer.backend.ops.load_backend import BACKEND
+
+if BACKEND == 'tensorflow':
+ from .tensorflow_data import *
+ from .tensorflow_image import *
+
+elif BACKEND == 'mindspore':
+ from .mindspore_data import *
+ from .mindspore_image import *
+
+elif BACKEND == 'dragon':
+ pass
+
+elif BACKEND == 'paddle':
+ pass
+
+else:
+ raise NotImplementedError("This backend is not supported")
+
diff --git a/tensorlayer/dataflow/dataflow_examples.py b/tensorlayer/dataflow/dataflow_examples.py
new file mode 100644
index 0000000..2bee246
--- /dev/null
+++ b/tensorlayer/dataflow/dataflow_examples.py
@@ -0,0 +1,56 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import tensorlayer as tl
+from tensorlayer.dataflow import Dataset
+import numpy as np
+
+X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3), plotable=False)
+
+
+def generator_train():
+ inputs = X_train
+ targets = y_train
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+ for _input, _target in zip(inputs, targets):
+ # yield _input.encode('utf-8'), _target.encode('utf-8')
+ yield (_input, np.array(_target))
+
+
+batch_size = 128
+shuffle_buffer_size = 128
+n_epoch = 10
+
+import tensorflow as tf
+
+
+def _map_fn_train(img, target):
+ # 1. Randomly crop a [height, width] section of the image.
+ img = tf.image.random_crop(img, [24, 24, 3])
+ # 2. Randomly flip the image horizontally.
+ img = tf.image.random_flip_left_right(img)
+ # 3. Randomly change brightness.
+ img = tf.image.random_brightness(img, max_delta=63)
+ # 4. Randomly change contrast.
+ img = tf.image.random_contrast(img, lower=0.2, upper=1.8)
+ # 5. Subtract off the mean and divide by the variance of the pixels.
+ img = tf.image.per_image_standardization(img)
+ target = tf.reshape(target, ())
+ return img, target
+
+
+import multiprocessing
+train_ds = Dataset.from_generator(
+ generator=generator_train, output_types=(tl.float32, tl.int32)
+) # , output_shapes=((24, 24, 3), (1)))
+
+train_ds = train_ds.map(_map_fn_train, num_parallel_calls=multiprocessing.cpu_count())
+
+train_ds = train_ds.repeat(n_epoch)
+train_ds = train_ds.shuffle(shuffle_buffer_size)
+train_ds = train_ds.prefetch(buffer_size=4096)
+train_ds = train_ds.batch(batch_size)
+
+for X_batch, y_batch in train_ds:
+ print(X_batch.shape, y_batch.shape)
diff --git a/tensorlayer/dataflow/image/__init__.py b/tensorlayer/dataflow/image/__init__.py
new file mode 100644
index 0000000..df05229
--- /dev/null
+++ b/tensorlayer/dataflow/image/__init__.py
@@ -0,0 +1,2 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
diff --git a/tensorlayer/dataflow/mindspore_data.py b/tensorlayer/dataflow/mindspore_data.py
new file mode 100644
index 0000000..fab1261
--- /dev/null
+++ b/tensorlayer/dataflow/mindspore_data.py
@@ -0,0 +1,277 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import mindspore.dataset as ds
+import mindspore as ms
+from enum import Enum
+__all__ = [
+ 'Apply',
+ 'Batch',
+ 'Concat',
+ 'CsvDataset',
+ 'Filter',
+ 'Flat_map',
+ 'FromGenerator',
+ 'FromSlices',
+ 'Map',
+ 'Prefetch',
+ 'Repeat',
+ 'Shuffle',
+ 'Skip',
+ 'Take',
+ 'TextFlieDataset',
+ 'TFRecordDataset',
+]
+
+
+class Shuffle(str, Enum):
+ GLOBAL: str = "global"
+ FILES: str = "file"
+
+
+def Apply(dataset, transformation_func):
+
+ return dataset.apply(transformation_func)
+
+
+def Batch(
+ dataset, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None, inut_columns=None,
+ output_columns=None, column_order=None, pad_info=None
+):
+ '''
+ Combine batch_size number of consecutive rows into batches.
+ Parameters
+ ----------
+ dataset
+ batch_size
+ drop_remainder
+ num_parallel_workers
+ per_batch_map
+ inut_columns
+ output_columns
+ column_order
+ pad_info
+
+ Returns
+ -------
+
+ '''
+ return dataset.batch(
+ batch_size=batch_size, drop_remainder=drop_remainder, num_parallel_workers=num_parallel_workers,
+ per_batch_map=per_batch_map, input_columns=inut_columns, output_columns=output_columns,
+ column_order=column_order, pad_info=pad_info
+ )
+
+
+def Concat(dataset_1, dataset_2):
+
+ return dataset_1.concat(dataset_2)
+
+
+def CsvDataset(
+ file_pattern, batch_size=1, column_names=None, column_defaults=None, label_name=None, select_columns=None,
+ field_delim=',', use_quote_delim=True, na_value='', header=True, num_epochs=None, shuffle=Shuffle.GLOBAL,
+ shuffle_buffer_size=10000, shuffle_seed=None, prefetch_buffer_size=None, num_parallel_reads=None, sloppy=False,
+ num_rows_for_inference=100, compression_type=None, ignore_errors=False, numples_samples=None, num_shards=None,
+ shard_id=None, cache=None
+):
+ """
+ A source dataset that reads and parses comma-separated values (CSV) datasets.
+
+ Examples:
+ >>> import mindspore.dataset as dataset
+ >>>
+ >>> dataset_files = ["/path/to/1", "/path/to/2"] # contains 1 or multiple text files
+ >>> dataset = dataset.CSVDataset(dataset_files=dataset_files, column_names=['col1', 'col2', 'col3', 'col4'])
+ """
+ return ds.CSVDataset(
+ dataset_files=file_pattern, field_delim=field_delim, column_defaults=column_defaults, column_names=column_names,
+ num_samples=numples_samples, num_parallel_workers=num_parallel_reads, shuffle=shuffle, num_shards=num_shards,
+ shard_id=shard_id, cache=cache
+ )
+
+
+def Filter(dataset, predicate):
+
+ return dataset.filter(predicate)
+
+
+def Flat_map(dataset, map_func):
+
+ return dataset.flat_map(map_func)
+
+
+def FromGenerator(
+ generator, output_types, output_shapes=None, args=None, column_names=None, column_types=None, schema=None,
+ num_samples=None, num_parallel_workers=1, shuffle=None, sampler=None, num_shards=None, shard_id=None,
+ python_multiprocessing=True
+):
+
+ return ds.GeneratorDataset(
+ source=generator, column_names=column_names, column_types=column_types, schema=schema, num_samples=num_samples,
+ num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, num_shards=num_shards,
+ shard_id=shard_id, python_multiprocessing=python_multiprocessing
+ )
+
+
+def FromSlices(
+ tensor, column_names=None, num_samples=None, num_parallel_workers=1, shuffle=None, sampler=None, num_shards=None,
+ shard_id=None
+):
+
+ return ds.NumpySlicesDataset(
+ data=tensor, column_names=column_names, num_samples=num_samples, num_parallel_workers=num_parallel_workers,
+ shuffle=shuffle, sampler=sampler, num_shards=num_shards, shard_id=shard_id
+ )
+
+
+def Map(
+ dataset, map_func, num_parallel_calls=None, input_columns=None, output_columns=None, column_order=None,
+ num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None
+):
+ """ Maps map_func across the elements of this dataset.
+
+ Parameters
+ ----------
+ dataset : DataFlow
+ input DataFlow
+ map_func : function
+ A function mapping a dataset element to another dataset element.
+ num_parallel_calls
+
+ Returns
+ -------
+
+ """
+ return dataset.map(
+ operations=map_func, input_columns=input_columns, output_columns=output_columns, column_order=column_order,
+ num_parallel_workers=num_parallel_workers, python_multiprocessing=python_multiprocessing, cache=cache,
+ callbacks=callbacks
+ )
+
+
+def Prefetch(dataset, buffer_size):
+
+ batch_size = dataset.get_batch_size()
+ prefetch_size = batch_size * buffer_size
+
+ return dataset.config.set_prefetch_size(prefetch_size)
+
+
+
+def Repeat(dataset, count=None):
+
+
+ return dataset.repeat(count)
+
+
+def Shuffle(dataset, buffer_size, seed=None, reshuffle_each_iteration=None):
+
+ #dataset.config.set_seed(seed)
+
+ return dataset.shuffle(buffer_size)
+
+
+def Skip(dataset, count):
+ '''
+ Creates a Dataset that skips count elements from this dataset.
+ Parameters
+ ----------
+ dataset:
+ A dataset
+ count:
+ A tf.int64 scalar tf.Tensor, representing the number of elements of this dataset that should be skipped to form the new dataset.
+
+
+ Returns
+ -------
+
+ '''
+ return dataset.skip(count)
+
+
+def Take(dataset, count):
+ '''
+ Creates a Dataset with at most count elements from this dataset.
+ Parameters
+ ----------
+ dataset:
+ A dataset
+ count:
+ A tf.int64 scalar tf.Tensor, representing the number of elements of this dataset that should be taken to form the new dataset.
+ If count is -1, or if count is greater than the size of this dataset, the new dataset will contain all elements of this dataset.
+ Returns
+ -------
+
+ '''
+ return dataset.take(count)
+
+
+def TextFlieDataset(
+ filenames, compression_type=None, buffer_size=None, num_parallel_reads=None, num_samples=None, shuffle=None,
+ num_shards=None, shard_id=None, cache=None
+):
+ """
+ A source dataset that reads and parses datasets stored on disk in text format.
+ The generated dataset has one column ['text'].
+
+ Examples:
+ >>> import mindspore.dataset as dataset
+ >>>
+ >>> dataset_files = ["/path/to/1", "/path/to/2"] # contains 1 or multiple text files
+ >>> dataset = dataset.TextFileDataset(dataset_files=dataset_files)
+ """
+ if shuffle is None:
+ shuffle = Shuffle.GLOBAL
+ return ds.TextFileDataset(
+ dataset_files=filenames, num_samples=num_samples, num_parallel_workers=num_parallel_reads, shuffle=shuffle,
+ num_shards=num_shards, shard_id=shard_id, cache=cache
+ )
+
+
+def TFRecordDataset(
+ filenames, compression_type=None, buffer_size=None, num_parallel_reads=None, schema=None, columns_list=None,
+ num_samples=None, shuffle=None, num_shards=None, shard_id=None, shard_equal_rows=False, cache=None
+):
+ """
+ A source dataset that reads and parses datasets stored on disk in TFData format.
+
+ Examples:
+ >>> import mindspore.dataset as dataset
+ >>> import mindspore.common.dtype as mstype
+ >>>
+ >>> dataset_files = ["/path/to/1", "/path/to/2"] # contains 1 or multiple tf data files
+ >>>
+ >>> # 1) Get all rows from dataset_files with no explicit schema
+ >>> # The meta-data in the first row will be used as a schema.
+ >>> tfdataset = dataset.TFRecordDataset(dataset_files=dataset_files)
+ >>>
+ >>> # 2) Get all rows from dataset_files with user-defined schema
+ >>> schema = dataset.Schema()
+ >>> schema.add_column('col_1d', de_type=mindspore.int64, shape=[2])
+ >>> tfdataset = dataset.TFRecordDataset(dataset_files=dataset_files, schema=schema)
+ >>>
+ >>> # 3) Get all rows from dataset_files with schema file "./schema.json"
+ >>> tfdataset = dataset.TFRecordDataset(dataset_files=dataset_files, schema="./schema.json")
+ """
+ if shuffle is None:
+ shuffle = Shuffle.GLOBAL
+ return ds.TFRecordDataset(
+ dataset_files=filenames, schema=schema, columns_list=columns_list, num_samples=num_samples,
+ num_parallel_workers=num_parallel_reads, shuffle=shuffle, num_shards=num_shards, shard_id=shard_id,
+ shard_equal_rows=shard_equal_rows, cache=cache
+ )
+
+
+def Zip(datasets):
+ '''
+ Creates a Dataset by zipping together the given datasets.
+ Parameters
+ ----------
+ datasets:
+ A tuple of datasets to be zipped together.
+ Returns
+ -------
+
+ '''
+ return ds.zip(datasets)
diff --git a/tensorlayer/dataflow/mindspore_image.py b/tensorlayer/dataflow/mindspore_image.py
new file mode 100644
index 0000000..e4c1fd9
--- /dev/null
+++ b/tensorlayer/dataflow/mindspore_image.py
@@ -0,0 +1,305 @@
+import mindspore.dataset as ms
+import mindspore.dataset.vision.c_transforms as c_vision
+import mindspore.dataset.vision.py_transforms as py_vision
+import mindspore.dataset.vision.py_transforms_util as py_util
+import numpy as np
+from PIL import Image, ImageOps, ImageEnhance, __version__
+
+__all__ = [
+ 'CentralCrop', 'HsvToRgb', 'AdjustBrightness', 'AdjustContrast', 'AdjustHue', 'Crop', 'FlipHorizontal',
+ 'FlipVertical', 'GrayToRgb', 'RgbToGray', 'PadToBoundingBox'
+]
+
+augment_error_message = 'img should be PIL image. Got {}. Use Decode() for encoded data or ToPIL() for decoded data.'
+
+
+def CentralCrop(image, central_fraction=None, size=None):
+ '''
+
+ Parameters
+ ----------
+ image :
+ input Either a 3-D float Tensor of shape [height, width, depth],
+ or a 4-D Tensor of shape [batch_size, height, width, depth].
+ central_fraction :
+ float (0, 1], fraction of size to crop
+ size:
+ size (Union[int, sequence]) – The output size of the cropped image. If size is an integer, a square crop of size (size, size) is returned.
+ If size is a sequence of length 2, it should be (height, width).
+ Returns :
+ 3-D / 4-D float Tensor, as per the input.
+ -------
+ '''
+ if size is None and central_fraction is None:
+ raise ValueError('central_fraction and size can not be both None')
+
+ if size is None:
+ outshape = np.shape(image)
+ if len(outshape) == 3:
+ h_axis = 0
+ w_axis = 1
+ elif len(outshape) == 4:
+ h_axis = 1
+ w_axis = 2
+
+ height = outshape[h_axis]
+ width = outshape[w_axis]
+
+ target_height = height * central_fraction
+ target_width = width * central_fraction
+
+ size = (target_height, target_width)
+
+ return py_util.center_crop(image, size)
+
+
+def HsvToRgb(image, is_hwc=True):
+
+ image = np.asarray(image)
+
+ return py_util.hsv_to_rgbs(image, is_hwc=is_hwc)
+
+
+def AdjustBrightness(image, factor):
+ '''
+
+ Parameters
+ ----------
+ image:
+ input NumPy image array or PIL image
+ factor:
+ factor should be in the range (-1,1)
+ Returns:
+ -------
+ np darray image
+ '''
+
+ image = np.asarray(image)
+ image = image / 255
+ image = image + factor
+ index = np.where(image > 1)
+ image[index] = 1
+ index = np.where(image < 0)
+ image[index] = 0
+ image = image * 255
+
+ return image
+
+
+def AdjustContrast(image, factor):
+
+ if isinstance(image, np.ndarray):
+ image = Image.fromarray(image)
+ if not isinstance(image, Image.Image):
+ raise TypeError(augment_error_message.format(type(image)))
+
+ image = ImageEnhance.Contrast(image).enhance(factor)
+
+ image = np.array(image)
+
+ return image
+
+
+def AdjustHue(image, factor):
+
+ if isinstance(image, np.ndarray):
+ image = Image.fromarray(image)
+ if not isinstance(image, Image.Image):
+ raise TypeError(augment_error_message.format(type(image)))
+
+ image_hue_factor = factor
+ if not -1 <= image_hue_factor <= 1:
+ raise ValueError('image_hue_factor {} is not in [-1, 1].'.format(image_hue_factor))
+
+ if not isinstance(image, Image.Image):
+ raise TypeError(augment_error_message.format(type(image)))
+
+ mode = image.mode
+ if mode in {'L', '1', 'I', 'F'}:
+ return image
+
+ hue, saturation, value = image.convert('HSV').split()
+
+ np_hue = np.array(hue, dtype=np.uint8)
+
+ with np.errstate(over='ignore'):
+ np_hue += np.uint8(image_hue_factor * 255)
+ hue = Image.fromarray(np_hue, 'L')
+
+ image = Image.merge('HSV', (hue, saturation, value)).convert(mode)
+ return image
+
+
+def AdjustSaturation(image, factor):
+
+ if isinstance(image, np.ndarray):
+ image = Image.fromarray(image)
+ if not isinstance(image, Image.Image):
+ raise TypeError(augment_error_message.format(type(image)))
+
+ enhancer = ImageEnhance.Color(image)
+ image = enhancer.enhance(factor)
+ return image
+
+
+def Crop(image, offset_height, offset_width, target_height, target_width):
+
+ if isinstance(image, np.ndarray):
+ image = Image.fromarray(image)
+ if not isinstance(image, Image.Image):
+ raise TypeError(augment_error_message.format(type(image)))
+ image = np.array(
+ image.crop((offset_width, offset_height, offset_width + target_width, offset_width + target_height))
+ )
+ return image
+
+
+def FlipHorizontal(image):
+
+ if isinstance(image, np.ndarray):
+ image = Image.fromarray(image)
+ if not isinstance(image, Image.Image):
+ raise TypeError(augment_error_message.format(type(image)))
+
+ image = np.array(image.transpose(Image.FLIP_LEFT_RIGHT))
+
+ return image
+
+
+def FlipVertical(image):
+
+ if isinstance(image, np.ndarray):
+ image = Image.fromarray(image)
+ if not isinstance(image, Image.Image):
+ raise TypeError(augment_error_message.format(type(image)))
+
+ image = np.array(image.transpose(Image.FLIP_TOP_BOTTOM))
+
+ return image
+
+
+def GrayToRgb(image):
+
+ image = np.asarray(image)
+ shape = image.shape
+ output_image = np.zeros((shape[0], shape[1], 3), dtype=np.uint8)
+ if len(shape) == 3:
+ for i in range(3):
+ output_image[:, :, i] = image[:, :, 1]
+ elif len(shape) == 2:
+ for i in range(3):
+ output_image[:, :, i] = image
+
+ return output_image
+
+
+def RgbToGray(image):
+
+ if isinstance(image, np.ndarray):
+ image = Image.fromarray(image)
+ if not isinstance(image, Image.Image):
+ raise TypeError(augment_error_message.format(type(image)))
+ '''
+ 将彩色图像转换为灰度（模式“L”）时，库使用ITU-R 601-2 Luma转换：
+ L = R * 299/1000 + G * 587/1000 + B * 114/1000
+ '''
+ image = image.convert('L')
+ image = np.asarray(image)
+
+ return image
+
+
+def PadToBoundingBox(image, offset_height, offset_width, target_height, target_width):
+ '''
+
+ Parameters
+ ----------
+ image:
+ A PIL image
+ offset_height:
+ Number of rows of zeros to add on top.
+ offset_width:
+ Number of columns of zeros to add on the left.
+ target_height:
+ Height of output image.
+ target_width
+ Width of output image.
+ Returns
+ A numpy ndarray image
+ -------
+ '''
+
+ if offset_height < 0:
+ raise ValueError("offset_height must be >= 0")
+ if offset_width < 0:
+ raise ValueError("offset_width must be >= 0")
+ image = np.array(image)
+ shape = image.shape
+ top = offset_height
+ bottom = target_height - shape[0] - top
+ left = offset_width
+ right = target_width - shape[1] - left
+
+ if bottom < 0:
+ raise ValueError("target_height must be >= offset_height + height")
+
+ if right < 0:
+ raise ValueError("target_width must be >= offset_width + width")
+
+ return np.pad(image, ((top, bottom), (left, right), (0, 0)), mode='constant')
+
+
+def Standardization(image, mean=None, std=None, channel_mode=False):
+ '''
+
+ Parameters
+ ----------
+ image:
+ An n-D Tensor with at least 3 dimensions, the last 3 of which are the dimensions of each image.
+ mean:
+ List or tuple of mean values for each channel, with respect to channel order.
+ std:
+ List or tuple of standard deviations for each channel.
+ channel_mode:
+ Decide to implement standardization on whole image or each channel of image.
+ Returns:
+ A Tensor with the same shape and dtype as image.
+ -------
+ '''
+ image = np.array(image, dtype=np.float32)
+ num_shape = image.shape
+ if mean is not None and std is not None:
+ if len(mean) != len(std):
+ raise ValueError("Length of mean and std must be equal")
+ if len(mean) == 1:
+ mean = [mean[0]] * num_shape[2]
+ std = [std[0]] * num_shape[2]
+ mean = np.array(mean, dtype=image.dtype)
+ std = np.array(std, dtype=image.dtype)
+ return (image - mean[:, None, None]) / std[:, None, None]
+ elif mean is None and std is None:
+ if channel_mode:
+ num_pixels = num_shape[0] * num_shape[1]
+ image_mean = np.mean(image, axis=(0, 1))
+ stddev = np.std(image, axis=(0, 1))
+ min_sttdev = 1 / np.sqrt(num_pixels)
+ min_sttdev = [min_sttdev] * num_shape[2]
+ adjusted_sttdev = np.maximum(stddev, min_sttdev)
+
+ image -= image_mean
+ image = np.divide(image, adjusted_sttdev)
+ return image
+ else:
+ num_pixels = num_shape[0] * num_shape[1] * num_shape[2]
+ image_mean = np.mean(image, axis=(0, 1, 2))
+ image_mean = [image_mean] * 3
+ stddev = np.std(image, axis=(0, 1, 2))
+ min_sttdev = 1 / np.sqrt(num_pixels)
+ adjusted_sttdev = np.maximum(stddev, min_sttdev)
+ adjusted_sttdev = [adjusted_sttdev] * 3
+
+ image -= image_mean
+ image = np.divide(image, adjusted_sttdev)
+ return image
+ else:
+ raise ValueError('std and mean must both be None or not None')
diff --git a/tensorlayer/dataflow/tensorflow_data.py b/tensorlayer/dataflow/tensorflow_data.py
new file mode 100644
index 0000000..ce50c77
--- /dev/null
+++ b/tensorlayer/dataflow/tensorflow_data.py
@@ -0,0 +1,254 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import tensorflow as tf
+
+__all__ = [
+ 'Apply',
+ 'Batch',
+ 'Concat',
+ 'CsvDataset',
+ 'Filter',
+ 'Flat_map',
+ 'FromGenerator',
+ 'FromSlices',
+ 'Map',
+ 'Prefetch',
+ 'Repeat',
+ 'Shuffle',
+ 'Skip',
+ 'Take',
+ 'TextFlieDataset',
+ 'TFRecordDataset',
+ 'Zip',
+]
+
+
+def Apply(dataset, transformation_func):
+ """Applies a transformation function to this dataset.
+ `apply` enables chaining of custom `Dataset` transformations, which are
+ represented as functions that take one `Dataset` argument and return a
+ transformed `Dataset`.
+ >>> dataset = tf.data.Dataset.range(100)
+ >>> def dataset_fn(dataset):
+ ... return dataset.filter(lambda x: x < 5)
+ >>> dataset = dataset.apply(dataset_fn)
+ >>> list(dataset.as_numpy_iterator())
+ [0, 1, 2, 3, 4]
+ Args:
+ transformation_func: A function that takes one `Dataset` argument and
+ returns a `Dataset`.
+ Returns:
+ Dataset: The `Dataset` returned by applying `transformation_func` to this
+ dataset.
+ """
+ return dataset.apply(transformation_func)
+
+
+def Batch(dataset, batch_size, drop_remainder=False):
+ '''
+
+ Parameters
+ ----------
+ dataset
+ batch_size
+ drop_remainder
+
+ Returns
+ -------
+
+ '''
+ return dataset.batch(batch_size=batch_size, drop_remainder=drop_remainder)
+
+
+def Concat(dataset_1, dataset_2):
+
+ return dataset_1.concatenate(dataset_2)
+
+
+def CsvDataset(
+ file_pattern, batch_size=1, column_names=None, column_defaults=None, label_name=None, select_columns=None,
+ field_delim=',', use_quote_delim=True, na_value='', header=True, num_epochs=None, shuffle=True,
+ shuffle_buffer_size=10000, shuffle_seed=None, prefetch_buffer_size=None, num_parallel_reads=None, sloppy=False,
+ num_rows_for_inference=100, compression_type=None, ignore_errors=False, numples_samples=None, num_shards=None,
+ shard_id=None, cache=None
+):
+ """Reads CSV files into a dataset.
+ Reads CSV files into a dataset, where each element is a (features, labels)
+ tuple that corresponds to a batch of CSV rows. The features dictionary
+ maps feature column names to `Tensor`s containing the corresponding
+ feature data, and labels is a `Tensor` containing the batch's label data.
+ """
+ return tf.data.experimental.make_csv_dataset(
+ file_pattern, batch_size, column_names=None, column_defaults=None, label_name=None, select_columns=None,
+ field_delim=',', use_quote_delim=True, na_value='', header=True, num_epochs=None, shuffle=True,
+ shuffle_buffer_size=10000, shuffle_seed=None, prefetch_buffer_size=None, num_parallel_reads=None, sloppy=False,
+ num_rows_for_inference=100, compression_type=None, ignore_errors=False
+ )
+
+
+def Filter(dataset, predicate):
+ '''
+ Filters this dataset according to predicate.
+ Parameters
+ ----------
+ dataset :
+ A dataset
+ predicate :
+ A function mapping a dataset element to a boolean.
+ Returns :
+ The Dataset containing the elements of this dataset for which predicate is True.
+ -------
+
+ '''
+ return dataset.filter(predicate)
+
+
+def Flat_map(dataset, map_func):
+ '''
+ Maps map_func across this dataset and flattens the result.
+ Parameters
+ ----------
+ dataset:
+ A dataset
+ map_func
+ A function mapping a dataset element to a dataset.
+ Returns
+ A Dataset.
+ -------
+
+ '''
+ return dataset.flat_map(map_func)
+
+
+def FromGenerator(
+ generator, output_types, output_shapes=None, args=None, column_names=None, column_types=None, schema=None,
+ num_samples=None, num_parallel_workers=1, shuffle=None, sampler=None, num_shards=None, shard_id=None,
+ python_multiprocessing=True
+):
+ """Creates a `Dataset` whose elements are generated by `generator`.
+
+ generator:
+ A callable object
+ """
+ return tf.data.Dataset.from_generator(generator, output_types, output_shapes=output_shapes, args=args)
+
+
+def FromSlices(
+ tensor, column_names=None, num_samples=None, num_parallel_workers=1, shuffle=None, sampler=None, num_shards=None,
+ shard_id=None
+):
+
+ return tf.data.Dataset.from_tensor_slices(tensor)
+
+
+def Map(
+ dataset, map_func, num_parallel_calls=None, input_columns=None, output_columns=None, column_order=None,
+ num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None
+):
+ """ Maps map_func across the elements of this dataset.
+
+ Parameters
+ ----------
+ dataset : DataFlow
+ input DataFlow
+ map_func : function
+ A function mapping a dataset element to another dataset element.
+ num_parallel_calls
+
+ Returns
+ -------
+
+ """
+ return dataset.map(map_func, num_parallel_calls=num_parallel_calls)
+
+
+def Prefetch(dataset, buffer_size):
+ '''
+ Creates a Dataset that prefetches elements from this dataset.
+ Parameters
+ ----------
+ dataset: Dataflow
+ A dataset
+ buffer_size :
+ A tf.int64 scalar tf.Tensor, representing the maximum number of elements that will be buffered when prefetching.
+ Returns
+ A Dataset
+ -------
+
+ '''
+ return dataset.prefetch(buffer_size=buffer_size)
+
+
+def Repeat(dataset, count=None):
+ return dataset.repeat(count=count)
+
+
+def Shuffle(dataset, buffer_size, seed=None, reshuffle_each_iteration=None):
+ return dataset.shuffle(buffer_size, seed=seed, reshuffle_each_iteration=reshuffle_each_iteration)
+
+
+def Skip(dataset, count):
+ '''
+ Creates a Dataset that skips count elements from this dataset.
+ Parameters
+ ----------
+ dataset:
+ A dataset
+ count:
+ A tf.int64 scalar tf.Tensor, representing the number of elements of this dataset that should be skipped to form the new dataset.
+ If count is greater than the size of this dataset, the new dataset will contain no elements.
+ If count is -1, skips the entire dataset.
+
+ Returns
+ -------
+
+ '''
+ return dataset.skip(count)
+
+
+def Take(dataset, count):
+ '''
+ Creates a Dataset with at most count elements from this dataset.
+ Parameters
+ ----------
+ dataset:
+ A dataset
+ count:
+ A tf.int64 scalar tf.Tensor, representing the number of elements of this dataset that should be taken to form the new dataset.
+ If count is -1, or if count is greater than the size of this dataset, the new dataset will contain all elements of this dataset.
+ Returns
+ -------
+
+ '''
+ return dataset.take(count)
+
+
+def TextFlieDataset(
+ filenames, compression_type=None, buffer_size=None, num_parallel_reads=None, num_samples=None, shuffle=None,
+ num_shards=None, shard_id=None, cache=None
+):
+
+ return tf.data.TextLineDataset(filenames, compression_type, buffer_size, num_parallel_reads)
+
+
+def TFRecordDataset(
+ filenames, compression_type=None, buffer_size=None, num_parallel_reads=None, schema=None, columns_list=None,
+ num_samples=None, shuffle=None, num_shards=None, shard_id=None, shard_equal_rows=False, cache=None
+):
+
+ return tf.data.TFRecordDataset(filenames, compression_type, buffer_size, num_parallel_reads)
+
+
+def Zip(datasets):
+ '''
+ Creates a Dataset by zipping together the given datasets.
+ Parameters
+ ----------
+ datasets:
+ A tuple of datasets to be zipped together.
+ Returns
+ -------
+
+ '''
+ return tf.data.Dataset.zip(datasets)
diff --git a/tensorlayer/dataflow/tensorflow_image.py b/tensorlayer/dataflow/tensorflow_image.py
new file mode 100644
index 0000000..39419b9
--- /dev/null
+++ b/tensorlayer/dataflow/tensorflow_image.py
@@ -0,0 +1,200 @@
+import tensorflow as tf
+import numpy as np
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.ops.image_ops_impl import _AssertAtLeast3DImage
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops.image_ops_impl import convert_image_dtype
+__all__ = [
+ 'CentralCrop',
+ 'HsvToRgb',
+ 'AdjustBrightness',
+ 'AdjustContrast',
+ 'AdjustHue',
+ 'AdjustSaturation',
+ 'Crop',
+ 'FlipHorizontal',
+ 'FlipVertical',
+ 'GrayToRgb',
+ 'Standardization',
+]
+
+
+def CentralCrop(image, central_fraction=None, size=None):
+ '''
+
+ Parameters
+ ----------
+ image :
+ input Either a 3-D float Tensor of shape [height, width, depth],
+ or a 4-D Tensor of shape [batch_size, height, width, depth].
+ central_fraction :
+ float (0, 1], fraction of size to crop
+ size:
+ size (Union[int, sequence]) – The output size of the cropped image. If size is an integer, a square crop of size (size, size) is returned.
+ If size is a sequence of length 2, it should be (height, width).
+ Returns :
+ 3-D / 4-D float Tensor, as per the input.
+ -------
+ '''
+ if size is None and central_fraction is None:
+ raise ValueError('central_fraction and size can not be both None')
+
+ if central_fraction is None:
+ outshape = np.shape(image)
+ if len(outshape) == 3:
+ h_axis = 0
+ w_axis = 1
+ elif len(outshape) == 4:
+ h_axis = 1
+ w_axis = 2
+
+ if isinstance(size, int):
+ target_height = size
+ target_width = size
+ elif isinstance(size, tuple):
+ target_height = size[0]
+ target_width = size[1]
+
+ central_fraction = max(target_height // outshape[h_axis], target_width // outshape[w_axis])
+
+ return tf.image.central_crop(image, central_fraction)
+
+
+def HsvToRgb(image):
+
+ return tf.image.hsv_to_rgb(image)
+
+
+def AdjustBrightness(image, factor):
+
+ return tf.image.adjust_brightness(image, delta=factor)
+
+
+def AdjustContrast(image, factor):
+
+ return tf.image.adjust_contrast(image, contrast_factor=factor)
+
+
+def AdjustHue(image, factor):
+
+ return tf.image.adjust_hue(image, delta=factor)
+
+
+def AdjustSaturation(image, factor):
+
+ return tf.image.adjust_saturation(image, saturation_factor=factor)
+
+
+def Crop(image, offset_height, offset_width, target_height, target_width):
+ '''
+
+ Parameters
+ ----------
+ image:
+ A image or a batch of images
+ offset_height:
+ Vertical coordinate of the top-left corner of the result in the input.
+ offset_width:
+ Horizontal coordinate of the top-left corner of the result in the input.
+ target_height:
+ Height of the result.
+ target_width:
+ Width of the result.
+
+ Returns:
+ Output [batch, target_height, target_width, channels] or [target_height, target_width, channels]
+ -------
+ '''
+
+ return tf.image.crop_to_bounding_box(image, offset_height, offset_width, target_height, target_width)
+
+
+def FlipHorizontal(image):
+
+ return tf.image.flip_left_right(image)
+
+
+def FlipVertical(image):
+
+ return tf.image.flip_up_down(image)
+
+
+def GrayToRgb(image):
+
+ return tf.image.grayscale_to_rgb(image)
+
+
+def RgbToGray(image):
+
+ return tf.image.rgb_to_grayscale(image)
+
+
+def PadToBoundingBox(image, offset_height, offset_width, target_height, target_width):
+
+ return tf.image.pad_to_bounding_box(image, offset_height, offset_width, target_height, target_width)
+
+
+def Standardization(image, mean=None, std=None, channel_mode=False):
+ '''
+
+ Parameters
+ ----------
+ image:
+ An n-D Tensor with at least 3 dimensions, the last 3 of which are the dimensions of each image.
+ mean:
+ List or tuple of mean values for each channel, with respect to channel order.
+ std:
+ List or tuple of standard deviations for each channel.
+ channel_mode:
+ Decide to implement standardization on whole image or each channel of image.
+ Returns:
+ A Tensor with the same shape and dtype as image.
+ -------
+ '''
+ with ops.name_scope(None, 'Standardization', [image]) as scope:
+ image = ops.convert_to_tensor(image, name='image')
+ image = _AssertAtLeast3DImage(image)
+
+ orig_dtype = image.dtype
+ if orig_dtype not in [dtypes.float16, dtypes.float32]:
+ image = convert_image_dtype(image, dtypes.float32)
+
+ if mean is not None and std is not None:
+ mean = np.array(mean, dtype=np.float32)
+ std = np.array(std, dtype=np.float32)
+ image -= mean
+ image = math_ops.divide(image, std, name=scope)
+ return convert_image_dtype(image, orig_dtype, saturate=True)
+
+ elif mean is None and std is None:
+ if channel_mode:
+ num_pixels = math_ops.reduce_prod(array_ops.shape(image)[-3:-1])
+ #`num_pixels` is the number of elements in each channels of 'image'
+ image_mean = math_ops.reduce_mean(image, axis=[-2, -3], keepdims=True)
+ # `image_mean` is the mean of elements in each channels of 'image'
+
+ stddev = math_ops.reduce_std(image, axis=[-2, -3], keepdims=True)
+ min_stddev = math_ops.rsqrt(math_ops.cast(num_pixels, image.dtype))
+ adjusted_sttdev = math_ops.maximum(stddev, min_stddev)
+
+ image -= image_mean
+ image = math_ops.divide(image, adjusted_sttdev, name=scope)
+ return convert_image_dtype(image, orig_dtype, saturate=True)
+
+ else:
+ num_pixels = math_ops.reduce_prod(array_ops.shape(image)[-3:])
+ #`num_pixels` is the number of elements in `image`
+ image_mean = math_ops.reduce_mean(image, axis=[-1, -2, -3], keepdims=True)
+
+ # Apply a minimum normalization that protects us against uniform images.
+ stddev = math_ops.reduce_std(image, axis=[-1, -2, -3], keepdims=True)
+ min_stddev = math_ops.rsqrt(math_ops.cast(num_pixels, image.dtype))
+ adjusted_stddev = math_ops.maximum(stddev, min_stddev)
+
+ image -= image_mean
+ image = math_ops.divide(image, adjusted_stddev, name=scope)
+ return convert_image_dtype(image, orig_dtype, saturate=True)
+ else:
+ raise ValueError('std and mean must both be None or not None')
diff --git a/tensorlayer/db.py b/tensorlayer/db.py
new file mode 100644
index 0000000..129e251
--- /dev/null
+++ b/tensorlayer/db.py
@@ -0,0 +1,746 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+import pickle
+import sys
+import time
+from datetime import datetime
+
+import numpy as np
+import tensorflow as tf
+
+import gridfs
+import pymongo
+from tensorlayer import logging
+from tensorlayer.files import (
+ assign_weights, del_folder, exists_or_mkdir, load_hdf5_to_weights, save_weights_to_hdf5, static_graph2net
+)
+
+
+class TensorHub(object):
+ """It is a MongoDB based manager that help you to manage data, network architecture, parameters and logging.
+
+ Parameters
+ -------------
+ ip : str
+ Localhost or IP address.
+ port : int
+ Port number.
+ dbname : str
+ Database name.
+ username : str or None
+ User name, set to None if you do not need authentication.
+ password : str
+ Password.
+ project_name : str or None
+ Experiment key for this entire project, similar with the repository name of Github.
+
+ Attributes
+ ------------
+ ip, port, dbname and other input parameters : see above
+ See above.
+ project_name : str
+ The given project name, if no given, set to the script name.
+ db : mongodb client
+ See ``pymongo.MongoClient``.
+ """
+
+ # @deprecated_alias(db_name='dbname', user_name='username', end_support_version=2.1)
+ def __init__(
+ self, ip='localhost', port=27017, dbname='dbname', username='None', password='password', project_name=None
+ ):
+ self.ip = ip
+ self.port = port
+ self.dbname = dbname
+ self.username = username
+
+ print("[Database] Initializing ...")
+ # connect mongodb
+ client = pymongo.MongoClient(ip, port)
+ self.db = client[dbname]
+ if username is None:
+ print(username, password)
+ self.db.authenticate(username, password)
+ else:
+ print("[Database] No username given, it works if authentication is not required")
+ if project_name is None:
+ self.project_name = sys.argv[0].split('.')[0]
+ print("[Database] No project_name given, use {}".format(self.project_name))
+ else:
+ self.project_name = project_name
+
+ # define file system (Buckets)
+ self.dataset_fs = gridfs.GridFS(self.db, collection="datasetFilesystem")
+ self.model_fs = gridfs.GridFS(self.db, collection="modelfs")
+ # self.params_fs = gridfs.GridFS(self.db, collection="parametersFilesystem")
+ # self.architecture_fs = gridfs.GridFS(self.db, collection="architectureFilesystem")
+
+ print("[Database] Connected ")
+ _s = "[Database] Info:\n"
+ _s += " ip : {}\n".format(self.ip)
+ _s += " port : {}\n".format(self.port)
+ _s += " dbname : {}\n".format(self.dbname)
+ _s += " username : {}\n".format(self.username)
+ _s += " password : {}\n".format("*******")
+ _s += " project_name : {}\n".format(self.project_name)
+ self._s = _s
+ print(self._s)
+
+ def __str__(self):
+ """Print information of databset."""
+ return self._s
+
+ def _fill_project_info(self, args):
+ """Fill in project_name for all studies, architectures and parameters."""
+ return args.update({'project_name': self.project_name})
+
+ @staticmethod
+ def _serialization(ps):
+ """Serialize data."""
+ return pickle.dumps(ps, protocol=pickle.HIGHEST_PROTOCOL) # protocol=2)
+ # with open('_temp.pkl', 'wb') as file:
+ # return pickle.dump(ps, file, protocol=pickle.HIGHEST_PROTOCOL)
+
+ @staticmethod
+ def _deserialization(ps):
+ """Deseralize data."""
+ return pickle.loads(ps)
+
+ # =========================== MODELS ================================
+ def save_model(self, network=None, model_name='model', **kwargs):
+ """Save model architecture and parameters into database, timestamp will be added automatically.
+
+ Parameters
+ ----------
+ network : TensorLayer Model
+ TensorLayer Model instance.
+ model_name : str
+ The name/key of model.
+ kwargs : other events
+ Other events, such as name, accuracy, loss, step number and etc (optinal).
+
+ Examples
+ ---------
+ Save model architecture and parameters into database.
+ >>> db.save_model(net, accuracy=0.8, loss=2.3, name='second_model')
+
+ Load one model with parameters from database (run this in other script)
+ >>> net = db.find_top_model(accuracy=0.8, loss=2.3)
+
+ Find and load the latest model.
+ >>> net = db.find_top_model(sort=[("time", pymongo.DESCENDING)])
+ >>> net = db.find_top_model(sort=[("time", -1)])
+
+ Find and load the oldest model.
+ >>> net = db.find_top_model(sort=[("time", pymongo.ASCENDING)])
+ >>> net = db.find_top_model(sort=[("time", 1)])
+
+ Get model information
+ >>> net._accuracy
+ ... 0.8
+
+ Returns
+ ---------
+ boolean : True for success, False for fail.
+ """
+ kwargs.update({'model_name': model_name})
+ self._fill_project_info(kwargs) # put project_name into kwargs
+
+ # params = network.get_all_params()
+ params = network.all_weights
+
+ s = time.time()
+
+ # kwargs.update({'architecture': network.all_graphs, 'time': datetime.utcnow()})
+ kwargs.update({'architecture': network.config, 'time': datetime.utcnow()})
+
+ try:
+ params_id = self.model_fs.put(self._serialization(params))
+ kwargs.update({'params_id': params_id, 'time': datetime.utcnow()})
+ self.db.Model.insert_one(kwargs)
+ print("[Database] Save model: SUCCESS, took: {}s".format(round(time.time() - s, 2)))
+ return True
+ except Exception as e:
+ exc_type, exc_obj, exc_tb = sys.exc_info()
+ fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+ logging.info("{} {} {} {} {}".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))
+ print("[Database] Save model: FAIL")
+ return False
+
+ def find_top_model(self, sort=None, model_name='model', **kwargs):
+ """Finds and returns a model architecture and its parameters from the database which matches the requirement.
+
+ Parameters
+ ----------
+ sort : List of tuple
+ PyMongo sort comment, search "PyMongo find one sorting" and `collection level operations `__ for more details.
+ model_name : str or None
+ The name/key of model.
+ kwargs : other events
+ Other events, such as name, accuracy, loss, step number and etc (optinal).
+
+ Examples
+ ---------
+ - see ``save_model``.
+
+ Returns
+ ---------
+ network : TensorLayer Model
+ Note that, the returned network contains all information of the document (record), e.g. if you saved accuracy in the document, you can get the accuracy by using ``net._accuracy``.
+ """
+ # print(kwargs) # {}
+ kwargs.update({'model_name': model_name})
+ self._fill_project_info(kwargs)
+
+ s = time.time()
+
+ d = self.db.Model.find_one(filter=kwargs, sort=sort)
+
+ # _temp_file_name = '_find_one_model_ztemp_file'
+ if d is not None:
+ params_id = d['params_id']
+ graphs = d['architecture']
+ _datetime = d['time']
+ # exists_or_mkdir(_temp_file_name, False)
+ # with open(os.path.join(_temp_file_name, 'graph.pkl'), 'wb') as file:
+ # pickle.dump(graphs, file, protocol=pickle.HIGHEST_PROTOCOL)
+ else:
+ print("[Database] FAIL! Cannot find model: {}".format(kwargs))
+ return False
+ try:
+ params = self._deserialization(self.model_fs.get(params_id).read())
+ # TODO : restore model and load weights
+ network = static_graph2net(graphs)
+ assign_weights(weights=params, network=network)
+ # np.savez(os.path.join(_temp_file_name, 'params.npz'), params=params)
+ #
+ # network = load_graph_and_params(name=_temp_file_name, sess=sess)
+ # del_folder(_temp_file_name)
+
+ pc = self.db.Model.find(kwargs)
+ print(
+ "[Database] Find one model SUCCESS. kwargs:{} sort:{} save time:{} took: {}s".format(
+ kwargs, sort, _datetime, round(time.time() - s, 2)
+ )
+ )
+
+ # FIXME : not sure what's this for
+ # put all informations of model into the TL layer
+ # for key in d:
+ # network.__dict__.update({"_%s" % key: d[key]})
+
+ # check whether more parameters match the requirement
+ params_id_list = pc.distinct('params_id')
+ n_params = len(params_id_list)
+ if n_params != 1:
+ print(" Note that there are {} models match the kwargs".format(n_params))
+ return network
+ except Exception as e:
+ exc_type, exc_obj, exc_tb = sys.exc_info()
+ fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+ logging.info("{} {} {} {} {}".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))
+ return False
+
+ def delete_model(self, **kwargs):
+ """Delete model.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Find items to delete, leave it empty to delete all log.
+ """
+ self._fill_project_info(kwargs)
+ self.db.Model.delete_many(kwargs)
+ logging.info("[Database] Delete Model SUCCESS")
+
+ # =========================== DATASET ===============================
+ def save_dataset(self, dataset=None, dataset_name=None, **kwargs):
+ """Saves one dataset into database, timestamp will be added automatically.
+
+ Parameters
+ ----------
+ dataset : any type
+ The dataset you want to store.
+ dataset_name : str
+ The name of dataset.
+ kwargs : other events
+ Other events, such as description, author and etc (optinal).
+
+ Examples
+ ----------
+ Save dataset
+ >>> db.save_dataset([X_train, y_train, X_test, y_test], 'mnist', description='this is a tutorial')
+
+ Get dataset
+ >>> dataset = db.find_top_dataset('mnist')
+
+ Returns
+ ---------
+ boolean : Return True if save success, otherwise, return False.
+ """
+ self._fill_project_info(kwargs)
+ if dataset_name is None:
+ raise Exception("dataset_name is None, please give a dataset name")
+ kwargs.update({'dataset_name': dataset_name})
+
+ s = time.time()
+ try:
+ dataset_id = self.dataset_fs.put(self._serialization(dataset))
+ kwargs.update({'dataset_id': dataset_id, 'time': datetime.utcnow()})
+ self.db.Dataset.insert_one(kwargs)
+ # print("[Database] Save params: {} SUCCESS, took: {}s".format(file_name, round(time.time()-s, 2)))
+ print("[Database] Save dataset: SUCCESS, took: {}s".format(round(time.time() - s, 2)))
+ return True
+ except Exception as e:
+ exc_type, exc_obj, exc_tb = sys.exc_info()
+ fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+ logging.info("{} {} {} {} {}".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))
+ print("[Database] Save dataset: FAIL")
+ return False
+
+ def find_top_dataset(self, dataset_name=None, sort=None, **kwargs):
+ """Finds and returns a dataset from the database which matches the requirement.
+
+ Parameters
+ ----------
+ dataset_name : str
+ The name of dataset.
+ sort : List of tuple
+ PyMongo sort comment, search "PyMongo find one sorting" and `collection level operations `__ for more details.
+ kwargs : other events
+ Other events, such as description, author and etc (optinal).
+
+ Examples
+ ---------
+ Save dataset
+ >>> db.save_dataset([X_train, y_train, X_test, y_test], 'mnist', description='this is a tutorial')
+
+ Get dataset
+ >>> dataset = db.find_top_dataset('mnist')
+ >>> datasets = db.find_datasets('mnist')
+
+ Returns
+ --------
+ dataset : the dataset or False
+ Return False if nothing found.
+
+ """
+
+ self._fill_project_info(kwargs)
+ if dataset_name is None:
+ raise Exception("dataset_name is None, please give a dataset name")
+ kwargs.update({'dataset_name': dataset_name})
+
+ s = time.time()
+
+ d = self.db.Dataset.find_one(filter=kwargs, sort=sort)
+
+ if d is not None:
+ dataset_id = d['dataset_id']
+ else:
+ print("[Database] FAIL! Cannot find dataset: {}".format(kwargs))
+ return False
+ try:
+ dataset = self._deserialization(self.dataset_fs.get(dataset_id).read())
+ pc = self.db.Dataset.find(kwargs)
+ print("[Database] Find one dataset SUCCESS, {} took: {}s".format(kwargs, round(time.time() - s, 2)))
+
+ # check whether more datasets match the requirement
+ dataset_id_list = pc.distinct('dataset_id')
+ n_dataset = len(dataset_id_list)
+ if n_dataset != 1:
+ print(" Note that there are {} datasets match the requirement".format(n_dataset))
+ return dataset
+ except Exception as e:
+ exc_type, exc_obj, exc_tb = sys.exc_info()
+ fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+ logging.info("{} {} {} {} {}".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))
+ return False
+
+ def find_datasets(self, dataset_name=None, **kwargs):
+ """Finds and returns all datasets from the database which matches the requirement.
+ In some case, the data in a dataset can be stored separately for better management.
+
+ Parameters
+ ----------
+ dataset_name : str
+ The name/key of dataset.
+ kwargs : other events
+ Other events, such as description, author and etc (optional).
+
+ Returns
+ --------
+ params : the parameters, return False if nothing found.
+
+ """
+
+ self._fill_project_info(kwargs)
+ if dataset_name is None:
+ raise Exception("dataset_name is None, please give a dataset name")
+ kwargs.update({'dataset_name': dataset_name})
+
+ s = time.time()
+ pc = self.db.Dataset.find(kwargs)
+
+ if pc is not None:
+ dataset_id_list = pc.distinct('dataset_id')
+ dataset_list = []
+ for dataset_id in dataset_id_list: # you may have multiple Buckets files
+ tmp = self.dataset_fs.get(dataset_id).read()
+ dataset_list.append(self._deserialization(tmp))
+ else:
+ print("[Database] FAIL! Cannot find any dataset: {}".format(kwargs))
+ return False
+
+ print("[Database] Find {} datasets SUCCESS, took: {}s".format(len(dataset_list), round(time.time() - s, 2)))
+ return dataset_list
+
+ def delete_datasets(self, **kwargs):
+ """Delete datasets.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Find items to delete, leave it empty to delete all log.
+
+ """
+
+ self._fill_project_info(kwargs)
+ self.db.Dataset.delete_many(kwargs)
+ logging.info("[Database] Delete Dataset SUCCESS")
+
+ # =========================== LOGGING ===============================
+ def save_training_log(self, **kwargs):
+ """Saves the training log, timestamp will be added automatically.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Events, such as accuracy, loss, step number and etc.
+
+ Examples
+ ---------
+ >>> db.save_training_log(accuracy=0.33, loss=0.98)
+
+ """
+
+ self._fill_project_info(kwargs)
+ kwargs.update({'time': datetime.utcnow()})
+ _result = self.db.TrainLog.insert_one(kwargs)
+ _log = self._print_dict(kwargs)
+ logging.info("[Database] train log: " + _log)
+
+ def save_validation_log(self, **kwargs):
+ """Saves the validation log, timestamp will be added automatically.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Events, such as accuracy, loss, step number and etc.
+
+ Examples
+ ---------
+ >>> db.save_validation_log(accuracy=0.33, loss=0.98)
+
+ """
+
+ self._fill_project_info(kwargs)
+ kwargs.update({'time': datetime.utcnow()})
+ _result = self.db.ValidLog.insert_one(kwargs)
+ _log = self._print_dict(kwargs)
+ logging.info("[Database] valid log: " + _log)
+
+ def save_testing_log(self, **kwargs):
+ """Saves the testing log, timestamp will be added automatically.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Events, such as accuracy, loss, step number and etc.
+
+ Examples
+ ---------
+ >>> db.save_testing_log(accuracy=0.33, loss=0.98)
+
+ """
+
+ self._fill_project_info(kwargs)
+ kwargs.update({'time': datetime.utcnow()})
+ _result = self.db.TestLog.insert_one(kwargs)
+ _log = self._print_dict(kwargs)
+ logging.info("[Database] test log: " + _log)
+
+ def delete_training_log(self, **kwargs):
+ """Deletes training log.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Find items to delete, leave it empty to delete all log.
+
+ Examples
+ ---------
+ Save training log
+ >>> db.save_training_log(accuracy=0.33)
+ >>> db.save_training_log(accuracy=0.44)
+
+ Delete logs that match the requirement
+ >>> db.delete_training_log(accuracy=0.33)
+
+ Delete all logs
+ >>> db.delete_training_log()
+ """
+ self._fill_project_info(kwargs)
+ self.db.TrainLog.delete_many(kwargs)
+ logging.info("[Database] Delete TrainLog SUCCESS")
+
+ def delete_validation_log(self, **kwargs):
+ """Deletes validation log.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Find items to delete, leave it empty to delete all log.
+
+ Examples
+ ---------
+ - see ``save_training_log``.
+ """
+ self._fill_project_info(kwargs)
+ self.db.ValidLog.delete_many(kwargs)
+ logging.info("[Database] Delete ValidLog SUCCESS")
+
+ def delete_testing_log(self, **kwargs):
+ """Deletes testing log.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Find items to delete, leave it empty to delete all log.
+
+ Examples
+ ---------
+ - see ``save_training_log``.
+ """
+ self._fill_project_info(kwargs)
+ self.db.TestLog.delete_many(kwargs)
+ logging.info("[Database] Delete TestLog SUCCESS")
+
+ # def find_training_logs(self, **kwargs):
+ # pass
+ #
+ # def find_validation_logs(self, **kwargs):
+ # pass
+ #
+ # def find_testing_logs(self, **kwargs):
+ # pass
+
+ # =========================== Task ===================================
+ def create_task(self, task_name=None, script=None, hyper_parameters=None, saved_result_keys=None, **kwargs):
+ """Uploads a task to the database, timestamp will be added automatically.
+
+ Parameters
+ -----------
+ task_name : str
+ The task name.
+ script : str
+ File name of the python script.
+ hyper_parameters : dictionary
+ The hyper parameters pass into the script.
+ saved_result_keys : list of str
+ The keys of the task results to keep in the database when the task finishes.
+ kwargs : other parameters
+ Users customized parameters such as description, version number.
+
+ Examples
+ -----------
+ Uploads a task
+ >>> db.create_task(task_name='mnist', script='example/tutorial_mnist_simple.py', description='simple tutorial')
+
+ Finds and runs the latest task
+ >>> db.run_top_task(sort=[("time", pymongo.DESCENDING)])
+ >>> db.run_top_task(sort=[("time", -1)])
+
+ Finds and runs the oldest task
+ >>> db.run_top_task(sort=[("time", pymongo.ASCENDING)])
+ >>> db.run_top_task(sort=[("time", 1)])
+
+ """
+ if not isinstance(task_name, str): # is None:
+ raise Exception("task_name should be string")
+ if not isinstance(script, str): # is None:
+ raise Exception("script should be string")
+ if hyper_parameters is None:
+ hyper_parameters = {}
+ if saved_result_keys is None:
+ saved_result_keys = []
+
+ self._fill_project_info(kwargs)
+ kwargs.update({'time': datetime.utcnow()})
+ kwargs.update({'hyper_parameters': hyper_parameters})
+ kwargs.update({'saved_result_keys': saved_result_keys})
+
+ _script = open(script, 'rb').read()
+
+ kwargs.update({'status': 'pending', 'script': _script, 'result': {}})
+ self.db.Task.insert_one(kwargs)
+ logging.info("[Database] Saved Task - task_name: {} script: {}".format(task_name, script))
+
+ def run_top_task(self, task_name=None, sort=None, **kwargs):
+ """Finds and runs a pending task that in the first of the sorting list.
+
+ Parameters
+ -----------
+ task_name : str
+ The task name.
+ sort : List of tuple
+ PyMongo sort comment, search "PyMongo find one sorting" and `collection level operations `__ for more details.
+ kwargs : other parameters
+ Users customized parameters such as description, version number.
+
+ Examples
+ ---------
+ Monitors the database and pull tasks to run
+ >>> while True:
+ >>> print("waiting task from distributor")
+ >>> db.run_top_task(task_name='mnist', sort=[("time", -1)])
+ >>> time.sleep(1)
+
+ Returns
+ --------
+ boolean : True for success, False for fail.
+ """
+ if not isinstance(task_name, str): # is None:
+ raise Exception("task_name should be string")
+ self._fill_project_info(kwargs)
+ kwargs.update({'status': 'pending'})
+
+ # find task and set status to running
+ task = self.db.Task.find_one_and_update(kwargs, {'$set': {'status': 'running'}}, sort=sort)
+
+ # try:
+ # get task info e.g. hyper parameters, python script
+ if task is None:
+ logging.info("[Database] Find Task FAIL: key: {} sort: {}".format(task_name, sort))
+ return False
+ else:
+ logging.info("[Database] Find Task SUCCESS: key: {} sort: {}".format(task_name, sort))
+ _datetime = task['time']
+ _script = task['script']
+ _id = task['_id']
+ _hyper_parameters = task['hyper_parameters']
+ _saved_result_keys = task['saved_result_keys']
+ logging.info(" hyper parameters:")
+ for key in _hyper_parameters:
+ globals()[key] = _hyper_parameters[key]
+ logging.info(" {}: {}".format(key, _hyper_parameters[key]))
+ # run task
+ s = time.time()
+ logging.info("[Database] Start Task: key: {} sort: {} push time: {}".format(task_name, sort, _datetime))
+ _script = _script.decode('utf-8')
+ with tf.Graph().as_default(): # # as graph: # clear all TF graphs
+ exec(_script, globals())
+
+ # set status to finished
+ _ = self.db.Task.find_one_and_update({'_id': _id}, {'$set': {'status': 'finished'}})
+
+ # return results
+ __result = {}
+ for _key in _saved_result_keys:
+ logging.info(" result: {}={} {}".format(_key, globals()[_key], type(globals()[_key])))
+ __result.update({"%s" % _key: globals()[_key]})
+ _ = self.db.Task.find_one_and_update(
+ {'_id': _id}, {'$set': {
+ 'result': __result
+ }}, return_document=pymongo.ReturnDocument.AFTER
+ )
+ logging.info(
+ "[Database] Finished Task: task_name - {} sort: {} push time: {} took: {}s".format(
+ task_name, sort, _datetime,
+ time.time() - s
+ )
+ )
+ return True
+ # except Exception as e:
+ # exc_type, exc_obj, exc_tb = sys.exc_info()
+ # fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+ # logging.info("{} {} {} {} {}".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))
+ # logging.info("[Database] Fail to run task")
+ # # if fail, set status back to pending
+ # _ = self.db.Task.find_one_and_update({'_id': _id}, {'$set': {'status': 'pending'}})
+ # return False
+
+ def delete_tasks(self, **kwargs):
+ """Delete tasks.
+
+ Parameters
+ -----------
+ kwargs : logging information
+ Find items to delete, leave it empty to delete all log.
+
+ Examples
+ ---------
+ >>> db.delete_tasks()
+
+ """
+
+ self._fill_project_info(kwargs)
+ self.db.Task.delete_many(kwargs)
+ logging.info("[Database] Delete Task SUCCESS")
+
+ def check_unfinished_task(self, task_name=None, **kwargs):
+ """Finds and runs a pending task.
+
+ Parameters
+ -----------
+ task_name : str
+ The task name.
+ kwargs : other parameters
+ Users customized parameters such as description, version number.
+
+ Examples
+ ---------
+ Wait until all tasks finish in user's local console
+
+ >>> while not db.check_unfinished_task():
+ >>> time.sleep(1)
+ >>> print("all tasks finished")
+ >>> sess = tf.InteractiveSession()
+ >>> net = db.find_top_model(sess=sess, sort=[("test_accuracy", -1)])
+ >>> print("the best accuracy {} is from model {}".format(net._test_accuracy, net._name))
+
+ Returns
+ --------
+ boolean : True for success, False for fail.
+
+ """
+
+ if not isinstance(task_name, str): # is None:
+ raise Exception("task_name should be string")
+ self._fill_project_info(kwargs)
+
+ kwargs.update({'$or': [{'status': 'pending'}, {'status': 'running'}]})
+
+ # ## find task
+ # task = self.db.Task.find_one(kwargs)
+ task = self.db.Task.find(kwargs)
+
+ task_id_list = task.distinct('_id')
+ n_task = len(task_id_list)
+
+ if n_task == 0:
+ logging.info("[Database] No unfinished task - task_name: {}".format(task_name))
+ return False
+ else:
+
+ logging.info("[Database] Find {} unfinished task - task_name: {}".format(n_task, task_name))
+ return True
+
+ @staticmethod
+ def _print_dict(args):
+ string = ''
+ for key, value in args.items():
+ if key is not '_id':
+ string += str(key) + ": " + str(value) + " / "
+ return string
diff --git a/tensorlayer/decorators/__init__.py b/tensorlayer/decorators/__init__.py
new file mode 100644
index 0000000..ba8d5eb
--- /dev/null
+++ b/tensorlayer/decorators/__init__.py
@@ -0,0 +1,16 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+TensorLayer provides rich layer implementations trailed for
+various benchmarks and domain-specific problems. In addition, we also
+support transparent access to native TensorFlow parameters.
+For example, we provide not only layers for local response normalization, but also
+layers that allow user to apply ``tf.ops.lrn`` on ``network.outputs``.
+More functions can be found in `TensorFlow API `__.
+"""
+
+from .deprecated import deprecated
+from .deprecated_alias import deprecated_alias
+from .method_decorator import private_method, protected_method
+
+__all__ = ['deprecated', 'deprecated_alias', 'private_method', 'protected_method']
diff --git a/tensorlayer/decorators/deprecated.py b/tensorlayer/decorators/deprecated.py
new file mode 100644
index 0000000..ba5154d
--- /dev/null
+++ b/tensorlayer/decorators/deprecated.py
@@ -0,0 +1,61 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import functools
+import inspect
+import sys
+
+import wrapt
+
+from tensorlayer.decorators.utils import (
+ add_deprecation_notice_to_docstring, get_qualified_name, validate_deprecation_args
+)
+
+__all__ = ['deprecated']
+
+# Allow deprecation warnings to be silenced temporarily with a context manager.
+_PRINT_DEPRECATION_WARNINGS = True
+
+# Remember which deprecation warnings have been printed already.
+_PRINTED_WARNING = {}
+
+
+def deprecated(wrapped=None, date='', instructions='', warn_once=True):
+
+ if wrapped is None:
+ return functools.partial(deprecated, date=date, instructions=instructions, warn_once=warn_once)
+
+ @wrapt.decorator
+ def wrapper(wrapped, instance=None, args=None, kwargs=None):
+
+ validate_deprecation_args(date, instructions)
+
+ if _PRINT_DEPRECATION_WARNINGS:
+
+ class_or_func_name = get_qualified_name(wrapped)
+
+ if class_or_func_name not in _PRINTED_WARNING:
+ if warn_once:
+ _PRINTED_WARNING[class_or_func_name] = True
+
+ from tensorlayer import logging
+
+ logging.warning(
+ '%s: `%s.%s` (in file: %s) is deprecated and will be removed %s.\n'
+ 'Instructions for updating: %s\n' % (
+ "Class" if inspect.isclass(wrapped) else "Function", wrapped.__module__, class_or_func_name,
+ wrapped.__code__.co_filename, 'in a future version' if date is None else
+ ('after %s' % date), instructions
+ )
+ )
+
+ return wrapped(*args, **kwargs)
+
+ decorated = wrapper(wrapped)
+
+ if sys.version_info > (3, 0): # docstring can only be edited with Python 3
+ wrapt.FunctionWrapper.__setattr__(
+ decorated, "__doc__", add_deprecation_notice_to_docstring(wrapped.__doc__, date, instructions)
+ )
+
+ return decorated
diff --git a/tensorlayer/decorators/deprecated_alias.py b/tensorlayer/decorators/deprecated_alias.py
new file mode 100644
index 0000000..acd2c0f
--- /dev/null
+++ b/tensorlayer/decorators/deprecated_alias.py
@@ -0,0 +1,46 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import functools
+import warnings
+
+from tensorlayer import logging
+
+
+def deprecated_alias(end_support_version, **aliases):
+
+ def deco(f):
+
+ @functools.wraps(f)
+ def wrapper(*args, **kwargs):
+
+ try:
+ func_name = "{}.{}".format(args[0].__class__.__name__, f.__name__)
+ except (NameError, IndexError):
+ func_name = f.__name__
+
+ rename_kwargs(kwargs, aliases, end_support_version, func_name)
+
+ return f(*args, **kwargs)
+
+ return wrapper
+
+ return deco
+
+
+def rename_kwargs(kwargs, aliases, end_support_version, func_name):
+
+ for alias, new in aliases.items():
+
+ if alias in kwargs:
+
+ if new in kwargs:
+ raise TypeError('{}() received both {} and {}'.format(func_name, alias, new))
+
+ warnings.warn('{}() - {} is deprecated; use {}'.format(func_name, alias, new), DeprecationWarning)
+ logging.warning(
+ "DeprecationWarning: {}(): "
+ "`{}` argument is deprecated and will be removed in version {}, "
+ "please change for `{}.`".format(func_name, alias, end_support_version, new)
+ )
+ kwargs[new] = kwargs.pop(alias)
diff --git a/tensorlayer/decorators/method_decorator.py b/tensorlayer/decorators/method_decorator.py
new file mode 100644
index 0000000..5d26220
--- /dev/null
+++ b/tensorlayer/decorators/method_decorator.py
@@ -0,0 +1,44 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import inspect
+
+
+def private_method(func):
+ """Decorator for making an instance method private."""
+
+ def func_wrapper(*args, **kwargs):
+ """Decorator wrapper function."""
+ outer_frame = inspect.stack()[1][0]
+ if 'self' not in outer_frame.f_locals or outer_frame.f_locals['self'] is not args[0]:
+ raise RuntimeError('%s.%s is a private method' % (args[0].__class__.__name__, func.__name__))
+
+ return func(*args, **kwargs)
+
+ return func_wrapper
+
+
+def protected_method(func):
+ """Decorator for making an instance method private."""
+
+ def func_wrapper(*args, **kwargs):
+ """Decorator wrapper function."""
+ outer_frame = inspect.stack()[1][0]
+
+ caller = inspect.getmro(outer_frame.f_locals['self'].__class__)[:-1]
+ target = inspect.getmro(args[0].__class__)[:-1]
+
+ share_subsclass = False
+
+ for cls_ in target:
+ if issubclass(caller[0], cls_) or caller[0] is cls_:
+ share_subsclass = True
+ break
+
+ if ('self' not in outer_frame.f_locals or
+ outer_frame.f_locals['self'] is not args[0]) and (not share_subsclass):
+ raise RuntimeError('%s.%s is a protected method' % (args[0].__class__.__name__, func.__name__))
+
+ return func(*args, **kwargs)
+
+ return func_wrapper
diff --git a/tensorlayer/decorators/utils.py b/tensorlayer/decorators/utils.py
new file mode 100644
index 0000000..c0d294e
--- /dev/null
+++ b/tensorlayer/decorators/utils.py
@@ -0,0 +1,122 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+NOTE: DO NOT REMOVE THESE FILES. They are copied from Tensorflow repository and are necessary to build the library without installing TF.
+
+Source: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/util
+
+They replace the following imports:
+>>> from tensorflow.python.util import decorator_utils
+>>> from tensorflow.python.util.deprecation import _validate_deprecation_args
+"""
+
+import re
+import sys
+
+__all__ = ["add_deprecation_notice_to_docstring", "get_qualified_name", "validate_deprecation_args"]
+
+
+def add_deprecation_notice_to_docstring(doc, date, instructions):
+ return _add_deprecated_function_notice_to_docstring(doc, date, instructions)
+
+
+def get_qualified_name(function):
+ # Python 3
+ if hasattr(function, '__qualname__'):
+ return function.__qualname__
+
+ # Python 2
+ if hasattr(function, 'im_class'):
+ return function.im_class.__name__ + '.' + function.__name__
+ return function.__name__
+
+
+def validate_deprecation_args(date, instructions):
+ if date is not None and not re.match(r'20\d\d-[01]\d-[0123]\d', date):
+ raise ValueError('Date must be YYYY-MM-DD.')
+ if not instructions:
+ raise ValueError('Don\'t deprecate things without conversion instructions!')
+
+
+def _add_deprecated_function_notice_to_docstring(doc, date, instructions):
+ """Adds a deprecation notice to a docstring for deprecated functions."""
+
+ if instructions:
+ deprecation_message = """
+ .. warning::
+ **THIS FUNCTION IS DEPRECATED:** It will be removed after %s.
+ *Instructions for updating:* %s.
+ """ % (('in a future version' if date is None else ('after %s' % date)), instructions)
+
+ else:
+ deprecation_message = """
+ .. warning::
+ **THIS FUNCTION IS DEPRECATED:** It will be removed after %s.
+ """ % (('in a future version' if date is None else ('after %s' % date)))
+
+ main_text = [deprecation_message]
+
+ return _add_notice_to_docstring(doc, 'DEPRECATED FUNCTION', main_text)
+
+
+def _add_notice_to_docstring(doc, no_doc_str, notice):
+ """Adds a deprecation notice to a docstring."""
+ if not doc:
+ lines = [no_doc_str]
+
+ else:
+ lines = _normalize_docstring(doc).splitlines()
+
+ notice = [''] + notice
+
+ if len(lines) > 1:
+ # Make sure that we keep our distance from the main body
+ if lines[1].strip():
+ notice.append('')
+
+ lines[1:1] = notice
+ else:
+ lines += notice
+
+ return '\n'.join(lines)
+
+
+def _normalize_docstring(docstring):
+ """Normalizes the docstring.
+
+ Replaces tabs with spaces, removes leading and trailing blanks lines, and
+ removes any indentation.
+
+ Copied from PEP-257:
+ https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation
+
+ Args:
+ docstring: the docstring to normalize
+
+ Returns:
+ The normalized docstring
+ """
+ if not docstring:
+ return ''
+ # Convert tabs to spaces (following the normal Python rules)
+ # and split into a list of lines:
+ lines = docstring.expandtabs().splitlines()
+ # Determine minimum indentation (first line doesn't count):
+ # (we use sys.maxsize because sys.maxint doesn't exist in Python 3)
+ indent = sys.maxsize
+ for line in lines[1:]:
+ stripped = line.lstrip()
+ if stripped:
+ indent = min(indent, len(line) - len(stripped))
+ # Remove indentation (first line is special):
+ trimmed = [lines[0].strip()]
+ if indent < sys.maxsize:
+ for line in lines[1:]:
+ trimmed.append(line[indent:].rstrip())
+ # Strip off trailing and leading blank lines:
+ while trimmed and not trimmed[-1]:
+ trimmed.pop()
+ while trimmed and not trimmed[0]:
+ trimmed.pop(0)
+ # Return a single string:
+ return '\n'.join(trimmed)
diff --git a/tensorlayer/distributed.py b/tensorlayer/distributed.py
new file mode 100644
index 0000000..3b426f8
--- /dev/null
+++ b/tensorlayer/distributed.py
@@ -0,0 +1,545 @@
+# -*- coding: utf-8 -*-
+
+import json
+import os
+import time
+
+import tensorflow as tf
+from tensorflow.python.training import session_run_hook
+
+from tensorlayer import logging
+from tensorlayer.decorators import deprecated
+from tensorlayer.lazy_imports import LazyImport
+
+hvd = LazyImport('horovod.tensorflow')
+
+__all__ = ['TaskSpecDef', 'TaskSpec', 'DistributedSession', 'StopAtTimeHook', 'LoadCheckpoint', 'Trainer']
+
+
+class Trainer(object):
+ """Trainer for neural networks in a distributed environment.
+
+ TensorLayer Trainer is a high-level training interface built on top of TensorFlow MonitoredSession and
+ `Horovod `__. It transparently scales the training of a TensorLayer model
+ from a single GPU to multiple GPUs that be placed on different machines in a single cluster.
+
+ To run the trainer, you will need to install Horovod on your machine. Check the installation script at
+ `tensorlayer/scripts/download_and_install_openmpi3_ubuntu.sh`
+
+ The minimal inputs to the Trainer include (1) a training dataset defined using the TensorFlow DataSet API,
+ and (2) a model build function given the inputs of the training dataset, and returns the neural network
+ to train, the loss function to minimize, and the names of the tensor to log during training, and (3)
+ an optimizer and its arguments.
+
+ The default parameter choices of Trainer is inspired by the Facebook paper:
+ `Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour `__
+
+ Parameters
+ ----------
+ training_dataset : class TensorFlow ``DataSet``
+ The training dataset which zips samples and labels. The trainer automatically
+ shards the training dataset based on the number of GPUs.
+ build_training_func : function
+ A function that builds the training operator. It takes the training dataset as an input,
+ and returns the neural network, the loss function and a dictionary that maps
+ string tags to tensors to log during training.
+ optimizer : class TensorFlow ``Optimizer``
+ The loss function optimizer. The trainer automatically linearly scale the learning rate based on
+ the number of GPUs.
+ optimizer_args : dict
+ The optimizer argument dictionary. It must contain a `learning_rate` field in type of float.
+ Note that the learning rate is linearly scaled according to the number of GPU by default.
+ You can disable it using the option `scaling_learning_rate`
+ batch_size : int
+ The training mini-batch size (i.e., number of samples per batch).
+ prefetch_size: int or None
+ The dataset prefetch buffer size. Set this parameter to overlap the GPU training and data preparation
+ if the data preparation is heavy.
+ checkpoint_dir : None or str
+ The path to the TensorFlow model checkpoint. Note that only one trainer master would checkpoints its model.
+ If None, checkpoint is disabled.
+ log_step_size : int
+ The trainer logs training information every N mini-batches (i.e., step size).
+ validation_dataset: None or class TensorFlow ``DataSet``
+ The optional validation dataset that zips samples and labels. Note that
+ only the trainer master needs to the validation often.
+ build_validation_func: None or function
+ The function that builds the validation operator. It returns the validation neural network (which
+ share the weights of the training network) and a custom number of validation metrics.
+ scaling_learning_rate: Boolean
+ Linearly scale the learning rate by the number of GPUs. Default is True.
+ This `linear scaling rule` is generally effective and is highly recommended by the practioners.
+ Check `Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour `__
+ max_iteration: int
+ The maximum iteration (i.e., mini-batch) to train.
+ The default is `math.inf`. You can set it to a small number to end the training earlier. This is
+ usually set for testing purpose.
+
+ Attributes
+ ----------
+ training_network : class TensorLayer ``Layer``
+ The training model.
+ session : class TensorFlow ``MonitoredTrainingSession``
+ The training session tha the Trainer wraps.
+ global_step : int
+ The number of training mini-batch by far.
+ validation_metrics : list of tuples
+ The validation metrics that zips the validation metric property and the average value.
+
+ Examples
+ --------
+ See `tutorial_mnist_distributed_trainer.py
+ `__.
+
+ """
+
+ def __init__(
+ self, training_dataset, build_training_func, optimizer, optimizer_args, batch_size=32, prefetch_size=None,
+ checkpoint_dir=None, scaling_learning_rate=True, log_step_size=1, validation_dataset=None,
+ build_validation_func=None, max_iteration=float('inf')
+ ):
+ # Initialize Horovod.
+ hvd.init()
+ self.is_master = hvd.rank() == 0
+ self._last_global_step = 0
+
+ if prefetch_size is None:
+ prefetch_size = batch_size
+
+ # Define the loss for validation dataset
+ if validation_dataset:
+ validation_dataset = validation_dataset.shard(num_shards=hvd.size(), index=hvd.rank()).batch(batch_size)
+ validation_dataset.prefetch(buffer_size=prefetch_size)
+ self._validation_iterator = validation_dataset.make_initializable_iterator()
+ next_example, next_label = self._validation_iterator.get_next()
+ _, self._validation_metrics = build_validation_func(next_example, next_label)
+ if not isinstance(self._validation_metrics, list):
+ self._validation_metrics = list(self._validation_metrics)
+ else:
+ self._validation_iterator = None
+ self._validation_metrics = None
+
+ # Get the shard of the dataset based on my local rank
+ training_dataset = training_dataset.shard(num_shards=hvd.size(), index=hvd.rank()).batch(batch_size)
+
+ training_dataset.prefetch(buffer_size=prefetch_size)
+ training_iterator = training_dataset.make_one_shot_iterator()
+ self._training_network, loss, log_tensors = build_training_func(*training_iterator.get_next())
+
+ # Adjust learning rate based on number of GPUs.
+ lr = optimizer_args['learning_rate']
+ optimizer_args['learning_rate'] = lr * hvd.size() if scaling_learning_rate else lr
+ opt = optimizer(**optimizer_args)
+
+ # Add Horovod Distributed Optimizer.
+ opt = hvd.DistributedOptimizer(opt)
+
+ self._global_step = tf.train.get_or_create_global_step()
+ if isinstance(log_tensors, list):
+ log_tensors.append(self._global_step)
+ else:
+ log_tensors['global_step'] = self._global_step
+ self._train_op = opt.minimize(loss, global_step=self._global_step)
+
+ hooks = [
+ # Horovod: BroadcastGlobalVariablesHook broadcasts initial variable states
+ # from rank 0 to all other processes. This is necessary to ensure consistent
+ # initialization of all workers when training is started with random weights
+ # or restored from a checkpoint.
+ hvd.BroadcastGlobalVariablesHook(0),
+
+ # Horovod: adjust number of steps based on number of GPUs.
+ tf.train.StopAtStepHook(last_step=max_iteration // hvd.size()),
+ tf.train.LoggingTensorHook(tensors=log_tensors, every_n_iter=log_step_size),
+ ]
+
+ # Pin GPU to be used to process local rank (one GPU per process)
+ config = tf.ConfigProto()
+ config.gpu_options.allow_growth = True
+ config.gpu_options.visible_device_list = str(hvd.local_rank())
+
+ # Save checkpoints only on worker 0 to prevent other workers from
+ # corrupting them.
+ checkpoint_dir = checkpoint_dir if self.is_master else None
+
+ # The MonitoredTrainingSession takes care of session initialization,
+ # restoring from a checkpoint, saving to a checkpoint, and closing when done
+ # or an error occurs.
+ self._sess = tf.train.MonitoredTrainingSession(checkpoint_dir=checkpoint_dir, hooks=hooks, config=config)
+
+ @property
+ def global_step(self):
+ if self._sess.should_stop():
+ return self._last_global_step
+ self._last_global_step = self._sess.run(self._global_step)
+ return self._last_global_step
+
+ @property
+ def session(self):
+ return self._sess
+
+ @property
+ def training_network(self):
+ return self._training_network
+
+ @property
+ def validation_metrics(self):
+ """A helper function to compute validation related metrics"""
+
+ if (self._validation_iterator is None) or (self._validation_metrics is None):
+ raise AttributeError('Validation is not setup.')
+
+ n = 0.0
+ metric_sums = [0.0] * len(self._validation_metrics)
+ self._sess.run(self._validation_iterator.initializer)
+ while True:
+ try:
+ metrics = self._sess.run(self._validation_metrics)
+ for i, m in enumerate(metrics):
+ metric_sums[i] += m
+ n += 1.0
+ except tf.errors.OutOfRangeError:
+ break
+ for i, m in enumerate(metric_sums):
+ metric_sums[i] = metric_sums[i] / n
+ return zip(self._validation_metrics, metric_sums)
+
+ def train_on_batch(self):
+ """Train a mini-batch."""
+ self._sess.run(self._train_op)
+
+ def train_and_validate_to_end(self, validate_step_size=50):
+ """A helper function that shows how to train and validate a model at the same time.
+
+ Parameters
+ ----------
+ validate_step_size : int
+ Validate the training network every N steps.
+
+ """
+ while not self._sess.should_stop():
+ self.train_on_batch() # Run a training step synchronously.
+ if self.global_step % validate_step_size == 0:
+ # logging.info("Average loss for validation dataset: %s" % self.get_validation_metrics())
+ log_str = 'step: %d, ' % self.global_step
+ for n, m in self.validation_metrics:
+ log_str += '%s: %f, ' % (n.name, m)
+ logging.info(log_str)
+
+
+@deprecated(date="2018-10-30", instructions="Using the TensorLayer distributed trainer.")
+class TaskSpecDef(object):
+ """Specification for a distributed task.
+
+ It contains the job name, index of the task,
+ the parameter servers and the worker servers. If you want to use the last worker
+ for continuous evaluation you can call the method `use_last_worker_as_evaluator`
+ which returns a new :class:`TaskSpecDef` object without the last worker in the
+ cluster specification.
+
+ Parameters
+ ----------
+ task_type : str
+ Task type. One of `master`, `worker` or `ps`.
+ index : int
+ The zero-based index of the task. Distributed training jobs will have a single
+ master task, one or more parameter servers, and one or more workers.
+ trial : int
+ The identifier of the trial being run.
+ ps_hosts : str OR list of str
+ A string with a coma separate list of hosts for the parameter servers
+ or a list of hosts.
+ worker_hosts : str OR list of str
+ A string with a coma separate list of hosts for the worker servers
+ or a list of hosts.
+ master : str
+ A string with the master hosts
+
+ Notes
+ ----------
+ master might not be included in TF_CONFIG and can be None. The shard_index is adjusted
+ in any case to assign 0 to master and >= 1 to workers.
+ This implementation doesn't support sparse arrays in the `TF_CONFIG` variable as the
+ official TensorFlow documentation shows, as it is not a supported by the json
+ definition.
+
+ References
+ ----------
+ - `ML-engine trainer considerations `__
+
+ """
+
+ def __init__(self, task_type='master', index=0, trial=None, ps_hosts=None, worker_hosts=None, master=None):
+ self.type = task_type
+ self._index = int(index)
+ self._cluster_spec = None
+ self.num_workers = 1
+ self.num_ps = 0
+ self.shard_index = int(index)
+ self._master = True
+ self.trial = trial
+ self.ps_hosts = ps_hosts
+ self.worker_hosts = worker_hosts
+ self.master = master
+ self._server = None
+
+ if ps_hosts and worker_hosts:
+ self.ps_hosts = ps_hosts if isinstance(ps_hosts, list) else ps_hosts.split(',')
+ self.num_ps = len(self.ps_hosts)
+ self.worker_hosts = worker_hosts if isinstance(worker_hosts, list) else worker_hosts.split(',')
+ if master is not None and len(master) > 0:
+ self._cluster_spec = tf.train.ClusterSpec(
+ {
+ 'ps': self.ps_hosts,
+ 'worker': self.worker_hosts,
+ 'master': master
+ }
+ )
+ # master is a worker too
+ self.num_workers = len(self.worker_hosts) + 1
+ if self.type == 'worker':
+ self.shard_index = self._index + 1
+ self._master = self.type == 'master'
+ else:
+ self._cluster_spec = tf.train.ClusterSpec({'ps': self.ps_hosts, 'worker': self.worker_hosts})
+ self.num_workers = len(self.worker_hosts)
+ if self.type == 'worker':
+ self.shard_index = self._index
+ self._master = self.type == 'worker' and self._index == 0
+
+ def is_ps(self):
+ """Returns true if this server is a parameter server"""
+ return self.type == 'ps'
+
+ def is_worker(self):
+ """Returns true if this server is a worker server"""
+ return self.type == 'worker'
+
+ def is_master(self):
+ """Returns true if this server is the master server"""
+ return self._master
+
+ def is_evaluator(self):
+ """Returns true if this server is the evaluator server"""
+ return self.type == 'worker' and self.num_workers == self._index
+
+ def device_fn(self):
+ """Returns the function with the specification to create the graph in this server"""
+ current_device = '/job:{}/task:{}'.format(self.type, self._index)
+ ps_devices = '/job:ps'
+ return tf.train.replica_device_setter(
+ ps_device=ps_devices, worker_device=current_device, cluster=self._cluster_spec
+ )
+
+ def create_server(self):
+ if self._server is None and self.ps_hosts and self.worker_hosts and not self.is_evaluator():
+ # create server and join if it is a parameter server
+ self._server = tf.train.Server(self._cluster_spec, job_name=self.type, task_index=self._index)
+ if self.is_ps():
+ self._server.join()
+
+ def target(self):
+ if self._server is None:
+ self.create_server()
+ if self._server is not None:
+ return self._server.target
+ else:
+ return None
+
+ def use_last_worker_as_evaluator(self):
+ """Returns a new :class:`TaskSpecDef` where the last worker has been removed from
+ the list of worker_hosts, so it is not used for training anymore. You can call
+ is_evaluator to know whether this server is the evaluator one or not.
+ In case there is only one server for training this method raises an exception, as
+ you cannot use any server for evaluation.
+
+ """
+ if self.num_workers <= 1:
+ raise Exception('You need more than one worker instance to use one as evaluator')
+
+ return TaskSpecDef(
+ task_type=self.type, index=self._index, trial=self.trial, ps_hosts=self.ps_hosts,
+ worker_hosts=self.worker_hosts[:-1], master=self.master
+ )
+
+
+@deprecated(date="2018-10-30", instructions="Using the TensorLayer distributed trainer.")
+def create_task_spec_def():
+ """Returns the a :class:`TaskSpecDef` based on the environment variables for distributed training.
+
+ References
+ ----------
+ - `ML-engine trainer considerations `__
+ - `TensorPort Distributed Computing `__
+
+ """
+ if 'TF_CONFIG' in os.environ:
+ # TF_CONFIG is used in ML-engine
+ env = json.loads(os.environ.get('TF_CONFIG', '{}'))
+ task_data = env.get('task', None) or {'type': 'master', 'index': 0}
+ cluster_data = env.get('cluster', None) or {'ps': None, 'worker': None, 'master': None}
+ return TaskSpecDef(
+ task_type=task_data['type'], index=task_data['index'],
+ trial=task_data['trial'] if 'trial' in task_data else None, ps_hosts=cluster_data['ps'],
+ worker_hosts=cluster_data['worker'], master=cluster_data['master'] if 'master' in cluster_data else None
+ )
+ elif 'JOB_NAME' in os.environ:
+ # JOB_NAME, TASK_INDEX, PS_HOSTS, WORKER_HOSTS and MASTER_HOST are used in TensorPort
+ return TaskSpecDef(
+ task_type=os.environ['JOB_NAME'], index=os.environ['TASK_INDEX'], ps_hosts=os.environ.get('PS_HOSTS', None),
+ worker_hosts=os.environ.get('WORKER_HOSTS', None), master=os.environ.get('MASTER_HOST', None)
+ )
+ else:
+ raise Exception('You need to setup TF_CONFIG or JOB_NAME to define the task.')
+
+
+@deprecated(date="2018-10-30", instructions="Using the TensorLayer distributed trainer.")
+def create_distributed_session(
+ task_spec=None, checkpoint_dir=None, scaffold=None, hooks=None, chief_only_hooks=None, save_checkpoint_secs=600,
+ save_summaries_steps=object(), save_summaries_secs=object(), config=None, stop_grace_period_secs=120,
+ log_step_count_steps=100
+):
+ """Creates a distributed session.
+
+ It calls `MonitoredTrainingSession` to create a :class:`MonitoredSession` for distributed training.
+
+ Parameters
+ ----------
+ task_spec : :class:`TaskSpecDef`.
+ The task spec definition from create_task_spec_def()
+ checkpoint_dir : str.
+ Optional path to a directory where to restore variables.
+ scaffold : ``Scaffold``
+ A `Scaffold` used for gathering or building supportive ops.
+ If not specified, a default one is created. It's used to finalize the graph.
+ hooks : list of ``SessionRunHook`` objects.
+ Optional
+ chief_only_hooks : list of ``SessionRunHook`` objects.
+ Activate these hooks if `is_chief==True`, ignore otherwise.
+ save_checkpoint_secs : int
+ The frequency, in seconds, that a checkpoint is saved
+ using a default checkpoint saver. If `save_checkpoint_secs` is set to
+ `None`, then the default checkpoint saver isn't used.
+ save_summaries_steps : int
+ The frequency, in number of global steps, that the
+ summaries are written to disk using a default summary saver. If both
+ `save_summaries_steps` and `save_summaries_secs` are set to `None`, then
+ the default summary saver isn't used. Default 100.
+ save_summaries_secs : int
+ The frequency, in secs, that the summaries are written
+ to disk using a default summary saver. If both `save_summaries_steps` and
+ `save_summaries_secs` are set to `None`, then the default summary saver
+ isn't used. Default not enabled.
+ config : ``tf.ConfigProto``
+ an instance of `tf.ConfigProto` proto used to configure the session.
+ It's the `config` argument of constructor of `tf.Session`.
+ stop_grace_period_secs : int
+ Number of seconds given to threads to stop after
+ `close()` has been called.
+ log_step_count_steps : int
+ The frequency, in number of global steps, that the
+ global step/sec is logged.
+
+ Examples
+ --------
+ A simple example for distributed training where all the workers use the same dataset:
+
+ >>> task_spec = TaskSpec()
+ >>> with tf.device(task_spec.device_fn()):
+ >>> tensors = create_graph()
+ >>> with tl.DistributedSession(task_spec=task_spec,
+ ... checkpoint_dir='/tmp/ckpt') as session:
+ >>> while not session.should_stop():
+ >>> session.run(tensors)
+
+ An example where the dataset is shared among the workers
+ (see https://www.tensorflow.org/programmers_guide/datasets):
+
+ >>> task_spec = TaskSpec()
+ >>> # dataset is a :class:`tf.data.Dataset` with the raw data
+ >>> dataset = create_dataset()
+ >>> if task_spec is not None:
+ >>> dataset = dataset.shard(task_spec.num_workers, task_spec.shard_index)
+ >>> # shuffle or apply a map function to the new sharded dataset, for example:
+ >>> dataset = dataset.shuffle(buffer_size=10000)
+ >>> dataset = dataset.batch(batch_size)
+ >>> dataset = dataset.repeat(num_epochs)
+ >>> # create the iterator for the dataset and the input tensor
+ >>> iterator = dataset.make_one_shot_iterator()
+ >>> next_element = iterator.get_next()
+ >>> with tf.device(task_spec.device_fn()):
+ >>> # next_element is the input for the graph
+ >>> tensors = create_graph(next_element)
+ >>> with tl.DistributedSession(task_spec=task_spec,
+ ... checkpoint_dir='/tmp/ckpt') as session:
+ >>> while not session.should_stop():
+ >>> session.run(tensors)
+
+ References
+ ----------
+ - `MonitoredTrainingSession `__
+
+ """
+ target = task_spec.target() if task_spec is not None else None
+ is_chief = task_spec.is_master() if task_spec is not None else True
+ return tf.train.MonitoredTrainingSession(
+ master=target, is_chief=is_chief, checkpoint_dir=checkpoint_dir, scaffold=scaffold,
+ save_checkpoint_secs=save_checkpoint_secs, save_summaries_steps=save_summaries_steps,
+ save_summaries_secs=save_summaries_secs, log_step_count_steps=log_step_count_steps,
+ stop_grace_period_secs=stop_grace_period_secs, config=config, hooks=hooks, chief_only_hooks=chief_only_hooks
+ )
+
+
+@deprecated(date="2018-10-30", instructions="Using the TensorLayer distributed trainer.")
+class StopAtTimeHook(session_run_hook.SessionRunHook):
+ """Hook that requests stop after a specified time.
+
+ Parameters
+ ----------
+ time_running: int
+ Maximum time running in seconds
+
+ """
+
+ def __init__(self, time_running):
+ self._time_running = time_running
+ self._end_time = 0
+
+ def begin(self):
+ self._end_time = time.time() + self._time_running
+
+ def after_run(self, run_context, run_values):
+ if time.time() > self._end_time:
+ run_context.request_stop()
+
+
+@deprecated(date="2018-10-30", instructions="Using the TensorLayer distributed trainer.")
+class LoadCheckpoint(session_run_hook.SessionRunHook):
+ """Hook that loads a checkpoint after the session is created.
+
+ >>> from tensorflow.python.ops import variables as tf_variables
+ >>> from tensorflow.python.training.monitored_session import SingularMonitoredSession
+ >>>
+ >>> tensors = create_graph()
+ >>> saver = tf.train.Saver(var_list=tf_variables.trainable_variables())
+ >>> checkpoint_hook = LoadCheckpoint(saver, my_checkpoint_file)
+ >>> with tf.SingularMonitoredSession(hooks=[checkpoint_hook]) as session:
+ >>> while not session.should_stop():
+ >>> session.run(tensors)
+
+ """
+
+ def __init__(self, saver, checkpoint):
+ self._saver = saver
+ self._checkpoint = checkpoint
+ self._loaded = False
+
+ def after_create_session(self, session, coord):
+ if not self._loaded:
+ self._loaded = True
+ self._saver.restore(self._checkpoint)
+
+
+# Alias
+TaskSpec = create_task_spec_def
+DistributedSession = create_distributed_session
diff --git a/tensorlayer/files/__init__.py b/tensorlayer/files/__init__.py
new file mode 100644
index 0000000..8d985af
--- /dev/null
+++ b/tensorlayer/files/__init__.py
@@ -0,0 +1,77 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+TensorLayer provides rich layer implementations trailed for
+various benchmarks and domain-specific problems. In addition, we also
+support transparent access to native TensorFlow parameters.
+For example, we provide not only layers for local response normalization, but also
+layers that allow user to apply ``tf.ops.lrn`` on ``network.outputs``.
+More functions can be found in `TensorFlow API `__.
+"""
+
+from tensorlayer.lazy_imports import LazyImport
+
+from .dataset_loaders.celebA_dataset import *
+from .dataset_loaders.cifar10_dataset import *
+from .dataset_loaders.cyclegan_dataset import *
+from .dataset_loaders.flickr_1M_dataset import *
+from .dataset_loaders.flickr_25k_dataset import *
+from .dataset_loaders.imdb_dataset import *
+from .dataset_loaders.matt_mahoney_dataset import *
+from .dataset_loaders.mnist_dataset import *
+from .dataset_loaders.mnist_fashion_dataset import *
+from .dataset_loaders.mpii_dataset import *
+from .dataset_loaders.nietzsche_dataset import *
+from .dataset_loaders.ptb_dataset import *
+from .dataset_loaders.voc_dataset import *
+from .dataset_loaders.wmt_en_fr_dataset import *
+from .utils import *
+
+__all__ = [
+ # Dataset Loaders
+ 'load_celebA_dataset',
+ 'load_cifar10_dataset',
+ 'load_cyclegan_dataset',
+ 'load_fashion_mnist_dataset',
+ 'load_flickr1M_dataset',
+ 'load_flickr25k_dataset',
+ 'load_imdb_dataset',
+ 'load_matt_mahoney_text8_dataset',
+ 'load_mnist_dataset',
+ 'load_mpii_pose_dataset',
+ 'load_nietzsche_dataset',
+ 'load_ptb_dataset',
+ 'load_voc_dataset',
+ 'load_wmt_en_fr_dataset',
+
+ # Util Functions
+ 'assign_params',
+ 'del_file',
+ 'del_folder',
+ 'download_file_from_google_drive',
+ 'exists_or_mkdir',
+ 'file_exists',
+ 'folder_exists',
+ 'load_and_assign_npz',
+ 'load_and_assign_npz_dict',
+ 'load_ckpt',
+ 'load_cropped_svhn',
+ 'load_file_list',
+ 'load_folder_list',
+ 'load_npy_to_any',
+ 'load_npz',
+ 'maybe_download_and_extract',
+ 'natural_keys',
+ 'npz_to_W_pdf',
+ 'read_file',
+ 'save_any_to_npy',
+ 'save_ckpt',
+ 'save_npz',
+ 'save_npz_dict',
+ 'load_and_assign_ckpt',
+ 'ckpt_to_npz_dict'
+ #'save_graph',
+ #'load_graph',
+ #'save_graph_and_params',
+ #'load_graph_and_params',
+]
diff --git a/tensorlayer/files/dataset_loaders/__init__.py b/tensorlayer/files/dataset_loaders/__init__.py
new file mode 100644
index 0000000..59a5551
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/__init__.py
@@ -0,0 +1,34 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from .celebA_dataset import *
+from .cifar10_dataset import *
+from .cyclegan_dataset import *
+from .flickr_1M_dataset import *
+from .flickr_25k_dataset import *
+from .imdb_dataset import *
+from .matt_mahoney_dataset import *
+from .mnist_dataset import *
+from .mnist_fashion_dataset import *
+from .mpii_dataset import *
+from .nietzsche_dataset import *
+from .ptb_dataset import *
+from .voc_dataset import *
+from .wmt_en_fr_dataset import *
+
+__all__ = [
+ 'load_celebA_dataset',
+ 'load_cifar10_dataset',
+ 'load_cyclegan_dataset',
+ 'load_fashion_mnist_dataset',
+ 'load_flickr1M_dataset',
+ 'load_flickr25k_dataset',
+ 'load_imdb_dataset',
+ 'load_matt_mahoney_text8_dataset',
+ 'load_mnist_dataset',
+ 'load_mpii_pose_dataset',
+ 'load_nietzsche_dataset',
+ 'load_ptb_dataset',
+ 'load_voc_dataset',
+ 'load_wmt_en_fr_dataset',
+]
diff --git a/tensorlayer/files/dataset_loaders/celebA_dataset.py b/tensorlayer/files/dataset_loaders/celebA_dataset.py
new file mode 100644
index 0000000..d5dc575
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/celebA_dataset.py
@@ -0,0 +1,43 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+import zipfile
+
+from tensorlayer import logging
+from tensorlayer.files.utils import (download_file_from_google_drive, exists_or_mkdir, load_file_list)
+
+__all__ = ['load_celebA_dataset']
+
+
+def load_celebA_dataset(path='data'):
+ """Load CelebA dataset
+
+ Return a list of image path.
+
+ Parameters
+ -----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/celebA/``.
+
+ """
+ data_dir = 'celebA'
+ filename, drive_id = "img_align_celeba.zip", "0B7EVK8r0v71pZjFTYXZWM3FlRnM"
+ save_path = os.path.join(path, filename)
+ image_path = os.path.join(path, data_dir)
+ if os.path.exists(image_path):
+ logging.info('[*] {} already exists'.format(save_path))
+ else:
+ exists_or_mkdir(path)
+ download_file_from_google_drive(drive_id, save_path)
+ zip_dir = ''
+ with zipfile.ZipFile(save_path) as zf:
+ zip_dir = zf.namelist()[0]
+ zf.extractall(path)
+ os.remove(save_path)
+ os.rename(os.path.join(path, zip_dir), image_path)
+
+ data_files = load_file_list(path=image_path, regx='\\.jpg', printable=False)
+ for i, _v in enumerate(data_files):
+ data_files[i] = os.path.join(image_path, data_files[i])
+ return data_files
diff --git a/tensorlayer/files/dataset_loaders/cifar10_dataset.py b/tensorlayer/files/dataset_loaders/cifar10_dataset.py
new file mode 100644
index 0000000..9af3f61
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/cifar10_dataset.py
@@ -0,0 +1,134 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+import pickle
+import sys
+
+import numpy as np
+
+from tensorlayer import logging
+from tensorlayer.files.utils import maybe_download_and_extract
+
+__all__ = ['load_cifar10_dataset']
+
+
+def load_cifar10_dataset(shape=(-1, 32, 32, 3), path='data', plotable=False):
+ """Load CIFAR-10 dataset.
+
+ It consists of 60000 32x32 colour images in 10 classes, with
+ 6000 images per class. There are 50000 training images and 10000 test images.
+
+ The dataset is divided into five training batches and one test batch, each with
+ 10000 images. The test batch contains exactly 1000 randomly-selected images from
+ each class. The training batches contain the remaining images in random order,
+ but some training batches may contain more images from one class than another.
+ Between them, the training batches contain exactly 5000 images from each class.
+
+ Parameters
+ ----------
+ shape : tupe
+ The shape of digit images e.g. (-1, 3, 32, 32) and (-1, 32, 32, 3).
+ path : str
+ The path that the data is downloaded to, defaults is ``data/cifar10/``.
+ plotable : boolean
+ Whether to plot some image examples, False as default.
+
+ Examples
+ --------
+ >>> X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))
+
+ References
+ ----------
+ - `CIFAR website `__
+ - `Data download link `__
+ - ``__
+
+ """
+ path = os.path.join(path, 'cifar10')
+ logging.info("Load or Download cifar10 > {}".format(path))
+
+ #Helper function to unpickle the data
+ def unpickle(file):
+ fp = open(file, 'rb')
+ if sys.version_info.major == 2:
+ data = pickle.load(fp)
+ elif sys.version_info.major == 3:
+ data = pickle.load(fp, encoding='latin-1')
+ else:
+ raise RuntimeError("Sys Version Unsupported")
+ fp.close()
+ return data
+
+ filename = 'cifar-10-python.tar.gz'
+ url = 'https://www.cs.toronto.edu/~kriz/'
+ #Download and uncompress file
+ maybe_download_and_extract(filename, path, url, extract=True)
+
+ #Unpickle file and fill in data
+ X_train = None
+ y_train = []
+ for i in range(1, 6):
+ data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', "data_batch_{}".format(i)))
+ if i == 1:
+ X_train = data_dic['data']
+ else:
+ X_train = np.vstack((X_train, data_dic['data']))
+ y_train += data_dic['labels']
+
+ test_data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', "test_batch"))
+ X_test = test_data_dic['data']
+ y_test = np.array(test_data_dic['labels'])
+
+ if shape == (-1, 3, 32, 32):
+ X_test = X_test.reshape(shape)
+ X_train = X_train.reshape(shape)
+ elif shape == (-1, 32, 32, 3):
+ X_test = X_test.reshape(shape, order='F')
+ X_train = X_train.reshape(shape, order='F')
+ X_test = np.transpose(X_test, (0, 2, 1, 3))
+ X_train = np.transpose(X_train, (0, 2, 1, 3))
+ else:
+ X_test = X_test.reshape(shape)
+ X_train = X_train.reshape(shape)
+
+ y_train = np.array(y_train)
+
+ if plotable:
+ logging.info('\nCIFAR-10')
+ import matplotlib.pyplot as plt
+ fig = plt.figure(1)
+
+ logging.info('Shape of a training image: X_train[0] %s' % X_train[0].shape)
+
+ plt.ion() # interactive mode
+ count = 1
+ for _ in range(10): # each row
+ for _ in range(10): # each column
+ _ = fig.add_subplot(10, 10, count)
+ if shape == (-1, 3, 32, 32):
+ # plt.imshow(X_train[count-1], interpolation='nearest')
+ plt.imshow(np.transpose(X_train[count - 1], (1, 2, 0)), interpolation='nearest')
+ # plt.imshow(np.transpose(X_train[count-1], (2, 1, 0)), interpolation='nearest')
+ elif shape == (-1, 32, 32, 3):
+ plt.imshow(X_train[count - 1], interpolation='nearest')
+ # plt.imshow(np.transpose(X_train[count-1], (1, 0, 2)), interpolation='nearest')
+ else:
+ raise Exception("Do not support the given 'shape' to plot the image examples")
+ plt.gca().xaxis.set_major_locator(plt.NullLocator())
+ plt.gca().yaxis.set_major_locator(plt.NullLocator())
+ count = count + 1
+ plt.draw() # interactive mode
+ plt.pause(3) # interactive mode
+
+ logging.info("X_train: %s" % X_train.shape)
+ logging.info("y_train: %s" % y_train.shape)
+ logging.info("X_test: %s" % X_test.shape)
+ logging.info("y_test: %s" % y_test.shape)
+
+ X_train = np.asarray(X_train, dtype=np.float32)
+ X_test = np.asarray(X_test, dtype=np.float32)
+ y_train = np.asarray(y_train, dtype=np.int32)
+ y_test = np.asarray(y_test, dtype=np.int32)
+
+ return X_train, y_train, X_test, y_test
diff --git a/tensorlayer/files/dataset_loaders/cyclegan_dataset.py b/tensorlayer/files/dataset_loaders/cyclegan_dataset.py
new file mode 100644
index 0000000..e327b3b
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/cyclegan_dataset.py
@@ -0,0 +1,58 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+
+import numpy as np
+
+from tensorlayer import logging, visualize
+from tensorlayer.files.utils import (del_file, folder_exists, load_file_list, maybe_download_and_extract)
+
+__all__ = ['load_cyclegan_dataset']
+
+
+def load_cyclegan_dataset(filename='summer2winter_yosemite', path='data'):
+ """Load images from CycleGAN's database, see `this link `__.
+
+ Parameters
+ ------------
+ filename : str
+ The dataset you want, see `this link `__.
+ path : str
+ The path that the data is downloaded to, defaults is `data/cyclegan`
+
+ Examples
+ ---------
+ >>> im_train_A, im_train_B, im_test_A, im_test_B = load_cyclegan_dataset(filename='summer2winter_yosemite')
+
+ """
+ path = os.path.join(path, 'cyclegan')
+ url = 'https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/'
+
+ if folder_exists(os.path.join(path, filename)) is False:
+ logging.info("[*] {} is nonexistent in {}".format(filename, path))
+ maybe_download_and_extract(filename + '.zip', path, url, extract=True)
+ del_file(os.path.join(path, filename + '.zip'))
+
+ def load_image_from_folder(path):
+ path_imgs = load_file_list(path=path, regx='\\.jpg', printable=False)
+ return visualize.read_images(path_imgs, path=path, n_threads=10, printable=False)
+
+ im_train_A = load_image_from_folder(os.path.join(path, filename, "trainA"))
+ im_train_B = load_image_from_folder(os.path.join(path, filename, "trainB"))
+ im_test_A = load_image_from_folder(os.path.join(path, filename, "testA"))
+ im_test_B = load_image_from_folder(os.path.join(path, filename, "testB"))
+
+ def if_2d_to_3d(images): # [h, w] --> [h, w, 3]
+ for i, _v in enumerate(images):
+ if len(images[i].shape) == 2:
+ images[i] = images[i][:, :, np.newaxis]
+ images[i] = np.tile(images[i], (1, 1, 3))
+ return images
+
+ im_train_A = if_2d_to_3d(im_train_A)
+ im_train_B = if_2d_to_3d(im_train_B)
+ im_test_A = if_2d_to_3d(im_test_A)
+ im_test_B = if_2d_to_3d(im_test_B)
+
+ return im_train_A, im_train_B, im_test_A, im_test_B
diff --git a/tensorlayer/files/dataset_loaders/flickr_1M_dataset.py b/tensorlayer/files/dataset_loaders/flickr_1M_dataset.py
new file mode 100644
index 0000000..f2e582a
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/flickr_1M_dataset.py
@@ -0,0 +1,116 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+
+from tensorlayer import logging, visualize
+from tensorlayer.files.utils import (
+ del_file, folder_exists, load_file_list, load_folder_list, maybe_download_and_extract, read_file
+)
+
+__all__ = ['load_flickr1M_dataset']
+
+
+def load_flickr1M_dataset(tag='sky', size=10, path="data", n_threads=50, printable=False):
+ """Load Flick1M dataset.
+
+ Returns a list of images by a given tag from Flickr1M dataset,
+ it will download Flickr1M from `the official website `__
+ at the first time you use it.
+
+ Parameters
+ ------------
+ tag : str or None
+ What images to return.
+ - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search `__.
+ - If you want to get all images, set to ``None``.
+
+ size : int
+ integer between 1 to 10. 1 means 100k images ... 5 means 500k images, 10 means all 1 million images. Default is 10.
+ path : str
+ The path that the data is downloaded to, defaults is ``data/flickr25k/``.
+ n_threads : int
+ The number of thread to read image.
+ printable : boolean
+ Whether to print infomation when reading images, default is ``False``.
+
+ Examples
+ ----------
+ Use 200k images
+
+ >>> images = tl.files.load_flickr1M_dataset(tag='zebra', size=2)
+
+ Use 1 Million images
+
+ >>> images = tl.files.load_flickr1M_dataset(tag='zebra')
+
+ """
+ import shutil
+
+ path = os.path.join(path, 'flickr1M')
+ logging.info("[Flickr1M] using {}% of images = {}".format(size * 10, size * 100000))
+ images_zip = [
+ 'images0.zip', 'images1.zip', 'images2.zip', 'images3.zip', 'images4.zip', 'images5.zip', 'images6.zip',
+ 'images7.zip', 'images8.zip', 'images9.zip'
+ ]
+ tag_zip = 'tags.zip'
+ url = 'http://press.liacs.nl/mirflickr/mirflickr1m/'
+
+ # download dataset
+ for image_zip in images_zip[0:size]:
+ image_folder = image_zip.split(".")[0]
+ # logging.info(path+"/"+image_folder)
+ if folder_exists(os.path.join(path, image_folder)) is False:
+ # logging.info(image_zip)
+ logging.info("[Flickr1M] {} is missing in {}".format(image_folder, path))
+ maybe_download_and_extract(image_zip, path, url, extract=True)
+ del_file(os.path.join(path, image_zip))
+ # os.system("mv {} {}".format(os.path.join(path, 'images'), os.path.join(path, image_folder)))
+ shutil.move(os.path.join(path, 'images'), os.path.join(path, image_folder))
+ else:
+ logging.info("[Flickr1M] {} exists in {}".format(image_folder, path))
+
+ # download tag
+ if folder_exists(os.path.join(path, "tags")) is False:
+ logging.info("[Flickr1M] tag files is nonexistent in {}".format(path))
+ maybe_download_and_extract(tag_zip, path, url, extract=True)
+ del_file(os.path.join(path, tag_zip))
+ else:
+ logging.info("[Flickr1M] tags exists in {}".format(path))
+
+ # 1. image path list
+ images_list = []
+ images_folder_list = []
+ for i in range(0, size):
+ images_folder_list += load_folder_list(path=os.path.join(path, 'images%d' % i))
+ images_folder_list.sort(key=lambda s: int(s.split('/')[-1])) # folder/images/ddd
+
+ for folder in images_folder_list[0:size * 10]:
+ tmp = load_file_list(path=folder, regx='\\.jpg', printable=False)
+ tmp.sort(key=lambda s: int(s.split('.')[-2])) # ddd.jpg
+ images_list.extend([os.path.join(folder, x) for x in tmp])
+
+ # 2. tag path list
+ tag_list = []
+ tag_folder_list = load_folder_list(os.path.join(path, "tags"))
+
+ # tag_folder_list.sort(key=lambda s: int(s.split("/")[-1])) # folder/images/ddd
+ tag_folder_list.sort(key=lambda s: int(os.path.basename(s)))
+
+ for folder in tag_folder_list[0:size * 10]:
+ tmp = load_file_list(path=folder, regx='\\.txt', printable=False)
+ tmp.sort(key=lambda s: int(s.split('.')[-2])) # ddd.txt
+ tmp = [os.path.join(folder, s) for s in tmp]
+ tag_list += tmp
+
+ # 3. select images
+ logging.info("[Flickr1M] searching tag: {}".format(tag))
+ select_images_list = []
+ for idx, _val in enumerate(tag_list):
+ tags = read_file(tag_list[idx]).split('\n')
+ if tag in tags:
+ select_images_list.append(images_list[idx])
+
+ logging.info("[Flickr1M] reading images with tag: {}".format(tag))
+ images = visualize.read_images(select_images_list, '', n_threads=n_threads, printable=printable)
+ return images
diff --git a/tensorlayer/files/dataset_loaders/flickr_25k_dataset.py b/tensorlayer/files/dataset_loaders/flickr_25k_dataset.py
new file mode 100644
index 0000000..8049a06
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/flickr_25k_dataset.py
@@ -0,0 +1,81 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+
+from tensorlayer import logging, visualize
+from tensorlayer.files.utils import (
+ del_file, folder_exists, load_file_list, maybe_download_and_extract, natural_keys, read_file
+)
+
+__all__ = ['load_flickr25k_dataset']
+
+
+def load_flickr25k_dataset(tag='sky', path="data", n_threads=50, printable=False):
+ """Load Flickr25K dataset.
+
+ Returns a list of images by a given tag from Flick25k dataset,
+ it will download Flickr25k from `the official website `__
+ at the first time you use it.
+
+ Parameters
+ ------------
+ tag : str or None
+ What images to return.
+ - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search `__.
+ - If you want to get all images, set to ``None``.
+
+ path : str
+ The path that the data is downloaded to, defaults is ``data/flickr25k/``.
+ n_threads : int
+ The number of thread to read image.
+ printable : boolean
+ Whether to print infomation when reading images, default is ``False``.
+
+ Examples
+ -----------
+ Get images with tag of sky
+
+ >>> images = tl.files.load_flickr25k_dataset(tag='sky')
+
+ Get all images
+
+ >>> images = tl.files.load_flickr25k_dataset(tag=None, n_threads=100, printable=True)
+
+ """
+ path = os.path.join(path, 'flickr25k')
+
+ filename = 'mirflickr25k.zip'
+ url = 'http://press.liacs.nl/mirflickr/mirflickr25k/'
+
+ # download dataset
+ if folder_exists(os.path.join(path, "mirflickr")) is False:
+ logging.info("[*] Flickr25k is nonexistent in {}".format(path))
+ maybe_download_and_extract(filename, path, url, extract=True)
+ del_file(os.path.join(path, filename))
+
+ # return images by the given tag.
+ # 1. image path list
+ folder_imgs = os.path.join(path, "mirflickr")
+ path_imgs = load_file_list(path=folder_imgs, regx='\\.jpg', printable=False)
+ path_imgs.sort(key=natural_keys)
+
+ # 2. tag path list
+ folder_tags = os.path.join(path, "mirflickr", "meta", "tags")
+ path_tags = load_file_list(path=folder_tags, regx='\\.txt', printable=False)
+ path_tags.sort(key=natural_keys)
+
+ # 3. select images
+ if tag is None:
+ logging.info("[Flickr25k] reading all images")
+ else:
+ logging.info("[Flickr25k] reading images with tag: {}".format(tag))
+ images_list = []
+ for idx, _v in enumerate(path_tags):
+ tags = read_file(os.path.join(folder_tags, path_tags[idx])).split('\n')
+ # logging.info(idx+1, tags)
+ if tag is None or tag in tags:
+ images_list.append(path_imgs[idx])
+
+ images = visualize.read_images(images_list, folder_imgs, n_threads=n_threads, printable=printable)
+ return images
diff --git a/tensorlayer/files/dataset_loaders/imdb_dataset.py b/tensorlayer/files/dataset_loaders/imdb_dataset.py
new file mode 100644
index 0000000..2967e7e
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/imdb_dataset.py
@@ -0,0 +1,115 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import gzip
+import os
+
+import numpy as np
+import six.moves.cPickle as pickle
+
+from tensorlayer.files.utils import maybe_download_and_extract
+
+__all__ = ['load_imdb_dataset']
+
+
+def load_imdb_dataset(
+ path='data', nb_words=None, skip_top=0, maxlen=None, test_split=0.2, seed=113, start_char=1, oov_char=2,
+ index_from=3
+):
+ """Load IMDB dataset.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/imdb/``.
+ nb_words : int
+ Number of words to get.
+ skip_top : int
+ Top most frequent words to ignore (they will appear as oov_char value in the sequence data).
+ maxlen : int
+ Maximum sequence length. Any longer sequence will be truncated.
+ seed : int
+ Seed for reproducible data shuffling.
+ start_char : int
+ The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character.
+ oov_char : int
+ Words that were cut out because of the num_words or skip_top limit will be replaced with this character.
+ index_from : int
+ Index actual words with this index and higher.
+
+ Examples
+ --------
+ >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(
+ ... nb_words=20000, test_split=0.2)
+ >>> print('X_train.shape', X_train.shape)
+ (20000,) [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
+ >>> print('y_train.shape', y_train.shape)
+ (20000,) [1 0 0 ..., 1 0 1]
+
+ References
+ -----------
+ - `Modified from keras. `__
+
+ """
+ path = os.path.join(path, 'imdb')
+
+ filename = "imdb.pkl"
+ url = 'https://s3.amazonaws.com/text-datasets/'
+ maybe_download_and_extract(filename, path, url)
+
+ if filename.endswith(".gz"):
+ f = gzip.open(os.path.join(path, filename), 'rb')
+ else:
+ f = open(os.path.join(path, filename), 'rb')
+
+ X, labels = pickle.load(f)
+ f.close()
+
+ np.random.seed(seed)
+ np.random.shuffle(X)
+ np.random.seed(seed)
+ np.random.shuffle(labels)
+
+ if start_char is not None:
+ X = [[start_char] + [w + index_from for w in x] for x in X]
+ elif index_from:
+ X = [[w + index_from for w in x] for x in X]
+
+ if maxlen:
+ new_X = []
+ new_labels = []
+ for x, y in zip(X, labels):
+ if len(x) < maxlen:
+ new_X.append(x)
+ new_labels.append(y)
+ X = new_X
+ labels = new_labels
+ if not X:
+ raise Exception(
+ 'After filtering for sequences shorter than maxlen=' + str(maxlen) + ', no sequence was kept. '
+ 'Increase maxlen.'
+ )
+ if not nb_words:
+ nb_words = max([max(x) for x in X])
+
+ # by convention, use 2 as OOV word
+ # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
+ if oov_char is not None:
+ X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
+ else:
+ nX = []
+ for x in X:
+ nx = []
+ for w in x:
+ if (w >= nb_words or w < skip_top):
+ nx.append(w)
+ nX.append(nx)
+ X = nX
+
+ X_train = np.array(X[:int(len(X) * (1 - test_split))])
+ y_train = np.array(labels[:int(len(X) * (1 - test_split))])
+
+ X_test = np.array(X[int(len(X) * (1 - test_split)):])
+ y_test = np.array(labels[int(len(X) * (1 - test_split)):])
+
+ return X_train, y_train, X_test, y_test
diff --git a/tensorlayer/files/dataset_loaders/matt_mahoney_dataset.py b/tensorlayer/files/dataset_loaders/matt_mahoney_dataset.py
new file mode 100644
index 0000000..17a3e08
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/matt_mahoney_dataset.py
@@ -0,0 +1,48 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+import zipfile
+
+from tensorlayer import logging
+from tensorlayer.files.utils import maybe_download_and_extract
+
+__all__ = ['load_matt_mahoney_text8_dataset']
+
+
+def load_matt_mahoney_text8_dataset(path='data'):
+ """Load Matt Mahoney's dataset.
+
+ Download a text file from Matt Mahoney's website
+ if not present, and make sure it's the right size.
+ Extract the first file enclosed in a zip file as a list of words.
+ This dataset can be used for Word Embedding.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/mm_test8/``.
+
+ Returns
+ --------
+ list of str
+ The raw text data e.g. [.... 'their', 'families', 'who', 'were', 'expelled', 'from', 'jerusalem', ...]
+
+ Examples
+ --------
+ >>> words = tl.files.load_matt_mahoney_text8_dataset()
+ >>> print('Data size', len(words))
+
+ """
+ path = os.path.join(path, 'mm_test8')
+ logging.info("Load or Download matt_mahoney_text8 Dataset> {}".format(path))
+
+ filename = 'text8.zip'
+ url = 'http://mattmahoney.net/dc/'
+ maybe_download_and_extract(filename, path, url, expected_bytes=31344016)
+
+ with zipfile.ZipFile(os.path.join(path, filename)) as f:
+ word_list = f.read(f.namelist()[0]).split()
+ for idx, _ in enumerate(word_list):
+ word_list[idx] = word_list[idx].decode()
+ return word_list
diff --git a/tensorlayer/files/dataset_loaders/mnist_dataset.py b/tensorlayer/files/dataset_loaders/mnist_dataset.py
new file mode 100644
index 0000000..4e1346d
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/mnist_dataset.py
@@ -0,0 +1,31 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from tensorlayer.files.utils import _load_mnist_dataset
+
+__all__ = ['load_mnist_dataset']
+
+
+def load_mnist_dataset(shape=(-1, 784), path='data'):
+ """Load the original mnist.
+
+ Automatically download MNIST dataset and return the training, validation and test set with 50000, 10000 and 10000 digit images respectively.
+
+ Parameters
+ ----------
+ shape : tuple
+ The shape of digit images (the default is (-1, 784), alternatively (-1, 28, 28, 1)).
+ path : str
+ The path that the data is downloaded to.
+
+ Returns
+ -------
+ X_train, y_train, X_val, y_val, X_test, y_test: tuple
+ Return splitted training/validation/test set respectively.
+
+ Examples
+ --------
+ >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1,784), path='datasets')
+ >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 28, 28, 1))
+ """
+ return _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/')
diff --git a/tensorlayer/files/dataset_loaders/mnist_fashion_dataset.py b/tensorlayer/files/dataset_loaders/mnist_fashion_dataset.py
new file mode 100644
index 0000000..c7f1bb9
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/mnist_fashion_dataset.py
@@ -0,0 +1,33 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from tensorlayer.files.utils import _load_mnist_dataset
+
+__all__ = ['load_fashion_mnist_dataset']
+
+
+def load_fashion_mnist_dataset(shape=(-1, 784), path='data'):
+ """Load the fashion mnist.
+
+ Automatically download fashion-MNIST dataset and return the training, validation and test set with 50000, 10000 and 10000 fashion images respectively, `examples `__.
+
+ Parameters
+ ----------
+ shape : tuple
+ The shape of digit images (the default is (-1, 784), alternatively (-1, 28, 28, 1)).
+ path : str
+ The path that the data is downloaded to.
+
+ Returns
+ -------
+ X_train, y_train, X_val, y_val, X_test, y_test: tuple
+ Return splitted training/validation/test set respectively.
+
+ Examples
+ --------
+ >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_fashion_mnist_dataset(shape=(-1,784), path='datasets')
+ >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_fashion_mnist_dataset(shape=(-1, 28, 28, 1))
+ """
+ return _load_mnist_dataset(
+ shape, path, name='fashion_mnist', url='http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/'
+ )
diff --git a/tensorlayer/files/dataset_loaders/mnist_utils.py b/tensorlayer/files/dataset_loaders/mnist_utils.py
new file mode 100644
index 0000000..b2f27b3
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/mnist_utils.py
@@ -0,0 +1,75 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import gzip
+import os
+
+import numpy as np
+
+from tensorlayer import logging
+from tensorlayer.files.utils import maybe_download_and_extract
+
+__all__ = ["_load_mnist_dataset"]
+
+
+def _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/'):
+ """A generic function to load mnist-like dataset.
+
+ Parameters:
+ ----------
+ shape : tuple
+ The shape of digit images.
+ path : str
+ The path that the data is downloaded to.
+ name : str
+ The dataset name you want to use(the default is 'mnist').
+ url : str
+ The url of dataset(the default is 'http://yann.lecun.com/exdb/mnist/').
+ """
+ path = os.path.join(path, name)
+
+ # Define functions for loading mnist-like data's images and labels.
+ # For convenience, they also download the requested files if needed.
+ def load_mnist_images(path, filename):
+ filepath = maybe_download_and_extract(filename, path, url)
+
+ logging.info(filepath)
+ # Read the inputs in Yann LeCun's binary format.
+ with gzip.open(filepath, 'rb') as f:
+ data = np.frombuffer(f.read(), np.uint8, offset=16)
+ # The inputs are vectors now, we reshape them to monochrome 2D images,
+ # following the shape convention: (examples, channels, rows, columns)
+ data = data.reshape(shape)
+ # The inputs come as bytes, we convert them to float32 in range [0,1].
+ # (Actually to range [0, 255/256], for compatibility to the version
+ # provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)
+ return data / np.float32(256)
+
+ def load_mnist_labels(path, filename):
+ filepath = maybe_download_and_extract(filename, path, url)
+ # Read the labels in Yann LeCun's binary format.
+ with gzip.open(filepath, 'rb') as f:
+ data = np.frombuffer(f.read(), np.uint8, offset=8)
+ # The labels are vectors of integers now, that's exactly what we want.
+ return data
+
+ # Download and read the training and test set images and labels.
+ logging.info("Load or Download {0} > {1}".format(name.upper(), path))
+ X_train = load_mnist_images(path, 'train-images-idx3-ubyte.gz')
+ y_train = load_mnist_labels(path, 'train-labels-idx1-ubyte.gz')
+ X_test = load_mnist_images(path, 't10k-images-idx3-ubyte.gz')
+ y_test = load_mnist_labels(path, 't10k-labels-idx1-ubyte.gz')
+
+ # We reserve the last 10000 training examples for validation.
+ X_train, X_val = X_train[:-10000], X_train[-10000:]
+ y_train, y_val = y_train[:-10000], y_train[-10000:]
+
+ # We just return all the arrays in order, as expected in main().
+ # (It doesn't matter how we do this as long as we can read them again.)
+ X_train = np.asarray(X_train, dtype=np.float32)
+ y_train = np.asarray(y_train, dtype=np.int32)
+ X_val = np.asarray(X_val, dtype=np.float32)
+ y_val = np.asarray(y_val, dtype=np.int32)
+ X_test = np.asarray(X_test, dtype=np.float32)
+ y_test = np.asarray(y_test, dtype=np.int32)
+ return X_train, y_train, X_val, y_val, X_test, y_test
diff --git a/tensorlayer/files/dataset_loaders/mpii_dataset.py b/tensorlayer/files/dataset_loaders/mpii_dataset.py
new file mode 100644
index 0000000..a6f88f6
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/mpii_dataset.py
@@ -0,0 +1,252 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+
+from tensorlayer import logging
+from tensorlayer.files.utils import (del_file, folder_exists, load_file_list, maybe_download_and_extract)
+
+__all__ = ['load_mpii_pose_dataset']
+
+
+def load_mpii_pose_dataset(path='data', is_16_pos_only=False):
+ """Load MPII Human Pose Dataset.
+
+ Parameters
+ -----------
+ path : str
+ The path that the data is downloaded to.
+ is_16_pos_only : boolean
+ If True, only return the peoples contain 16 pose keypoints. (Usually be used for single person pose estimation)
+
+ Returns
+ ----------
+ img_train_list : list of str
+ The image directories of training data.
+ ann_train_list : list of dict
+ The annotations of training data.
+ img_test_list : list of str
+ The image directories of testing data.
+ ann_test_list : list of dict
+ The annotations of testing data.
+
+ Examples
+ --------
+ >>> import pprint
+ >>> import tensorlayer as tl
+ >>> img_train_list, ann_train_list, img_test_list, ann_test_list = tl.files.load_mpii_pose_dataset()
+ >>> image = tl.vis.read_image(img_train_list[0])
+ >>> tl.vis.draw_mpii_pose_to_image(image, ann_train_list[0], 'image.png')
+ >>> pprint.pprint(ann_train_list[0])
+
+ References
+ -----------
+ - `MPII Human Pose Dataset. CVPR 14 `__
+ - `MPII Human Pose Models. CVPR 16 `__
+ - `MPII Human Shape, Poselet Conditioned Pictorial Structures and etc `__
+ - `MPII Keyponts and ID `__
+ """
+ path = os.path.join(path, 'mpii_human_pose')
+ logging.info("Load or Download MPII Human Pose > {}".format(path))
+
+ # annotation
+ url = "http://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/"
+ tar_filename = "mpii_human_pose_v1_u12_2.zip"
+ extracted_filename = "mpii_human_pose_v1_u12_2"
+ if folder_exists(os.path.join(path, extracted_filename)) is False:
+ logging.info("[MPII] (annotation) {} is nonexistent in {}".format(extracted_filename, path))
+ maybe_download_and_extract(tar_filename, path, url, extract=True)
+ del_file(os.path.join(path, tar_filename))
+
+ # images
+ url = "http://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/"
+ tar_filename = "mpii_human_pose_v1.tar.gz"
+ extracted_filename2 = "images"
+ if folder_exists(os.path.join(path, extracted_filename2)) is False:
+ logging.info("[MPII] (images) {} is nonexistent in {}".format(extracted_filename, path))
+ maybe_download_and_extract(tar_filename, path, url, extract=True)
+ del_file(os.path.join(path, tar_filename))
+
+ # parse annotation, format see http://human-pose.mpi-inf.mpg.de/#download
+ import scipy.io as sio
+ logging.info("reading annotations from mat file ...")
+ # mat = sio.loadmat(os.path.join(path, extracted_filename, "mpii_human_pose_v1_u12_1.mat"))
+
+ # def fix_wrong_joints(joint): # https://github.com/mitmul/deeppose/blob/master/datasets/mpii_dataset.py
+ # if '12' in joint and '13' in joint and '2' in joint and '3' in joint:
+ # if ((joint['12'][0] < joint['13'][0]) and
+ # (joint['3'][0] < joint['2'][0])):
+ # joint['2'], joint['3'] = joint['3'], joint['2']
+ # if ((joint['12'][0] > joint['13'][0]) and
+ # (joint['3'][0] > joint['2'][0])):
+ # joint['2'], joint['3'] = joint['3'], joint['2']
+ # return joint
+
+ ann_train_list = []
+ ann_test_list = []
+ img_train_list = []
+ img_test_list = []
+
+ def save_joints():
+ # joint_data_fn = os.path.join(path, 'data.json')
+ # fp = open(joint_data_fn, 'w')
+ mat = sio.loadmat(os.path.join(path, extracted_filename, "mpii_human_pose_v1_u12_1.mat"))
+
+ for _, (anno, train_flag) in enumerate( # all images
+ zip(mat['RELEASE']['annolist'][0, 0][0], mat['RELEASE']['img_train'][0, 0][0])):
+
+ img_fn = anno['image']['name'][0, 0][0]
+ train_flag = int(train_flag)
+
+ # print(i, img_fn, train_flag) # DEBUG print all images
+
+ if train_flag:
+ img_train_list.append(img_fn)
+ ann_train_list.append([])
+ else:
+ img_test_list.append(img_fn)
+ ann_test_list.append([])
+
+ head_rect = []
+ if 'x1' in str(anno['annorect'].dtype):
+ head_rect = zip(
+ [x1[0, 0] for x1 in anno['annorect']['x1'][0]], [y1[0, 0] for y1 in anno['annorect']['y1'][0]],
+ [x2[0, 0] for x2 in anno['annorect']['x2'][0]], [y2[0, 0] for y2 in anno['annorect']['y2'][0]]
+ )
+ else:
+ head_rect = [] # TODO
+
+ if 'annopoints' in str(anno['annorect'].dtype):
+ annopoints = anno['annorect']['annopoints'][0]
+ head_x1s = anno['annorect']['x1'][0]
+ head_y1s = anno['annorect']['y1'][0]
+ head_x2s = anno['annorect']['x2'][0]
+ head_y2s = anno['annorect']['y2'][0]
+
+ for annopoint, head_x1, head_y1, head_x2, head_y2 in zip(annopoints, head_x1s, head_y1s, head_x2s,
+ head_y2s):
+ # if annopoint != []:
+ # if len(annopoint) != 0:
+ if annopoint.size:
+ head_rect = [
+ float(head_x1[0, 0]),
+ float(head_y1[0, 0]),
+ float(head_x2[0, 0]),
+ float(head_y2[0, 0])
+ ]
+
+ # joint coordinates
+ annopoint = annopoint['point'][0, 0]
+ j_id = [str(j_i[0, 0]) for j_i in annopoint['id'][0]]
+ x = [x[0, 0] for x in annopoint['x'][0]]
+ y = [y[0, 0] for y in annopoint['y'][0]]
+ joint_pos = {}
+ for _j_id, (_x, _y) in zip(j_id, zip(x, y)):
+ joint_pos[int(_j_id)] = [float(_x), float(_y)]
+ # joint_pos = fix_wrong_joints(joint_pos)
+
+ # visibility list
+ if 'is_visible' in str(annopoint.dtype):
+ vis = [v[0] if v.size > 0 else [0] for v in annopoint['is_visible'][0]]
+ vis = dict([(k, int(v[0])) if len(v) > 0 else v for k, v in zip(j_id, vis)])
+ else:
+ vis = None
+
+ # if len(joint_pos) == 16:
+ if ((is_16_pos_only ==True) and (len(joint_pos) == 16)) or (is_16_pos_only == False):
+ # only use image with 16 key points / or use all
+ data = {
+ 'filename': img_fn,
+ 'train': train_flag,
+ 'head_rect': head_rect,
+ 'is_visible': vis,
+ 'joint_pos': joint_pos
+ }
+ # print(json.dumps(data), file=fp) # py3
+ if train_flag:
+ ann_train_list[-1].append(data)
+ else:
+ ann_test_list[-1].append(data)
+
+ # def write_line(datum, fp):
+ # joints = sorted([[int(k), v] for k, v in datum['joint_pos'].items()])
+ # joints = np.array([j for i, j in joints]).flatten()
+ #
+ # out = [datum['filename']]
+ # out.extend(joints)
+ # out = [str(o) for o in out]
+ # out = ','.join(out)
+ #
+ # print(out, file=fp)
+
+ # def split_train_test():
+ # # fp_test = open('data/mpii/test_joints.csv', 'w')
+ # fp_test = open(os.path.join(path, 'test_joints.csv'), 'w')
+ # # fp_train = open('data/mpii/train_joints.csv', 'w')
+ # fp_train = open(os.path.join(path, 'train_joints.csv'), 'w')
+ # # all_data = open('data/mpii/data.json').readlines()
+ # all_data = open(os.path.join(path, 'data.json')).readlines()
+ # N = len(all_data)
+ # N_test = int(N * 0.1)
+ # N_train = N - N_test
+ #
+ # print('N:{}'.format(N))
+ # print('N_train:{}'.format(N_train))
+ # print('N_test:{}'.format(N_test))
+ #
+ # np.random.seed(1701)
+ # perm = np.random.permutation(N)
+ # test_indices = perm[:N_test]
+ # train_indices = perm[N_test:]
+ #
+ # print('train_indices:{}'.format(len(train_indices)))
+ # print('test_indices:{}'.format(len(test_indices)))
+ #
+ # for i in train_indices:
+ # datum = json.loads(all_data[i].strip())
+ # write_line(datum, fp_train)
+ #
+ # for i in test_indices:
+ # datum = json.loads(all_data[i].strip())
+ # write_line(datum, fp_test)
+
+ save_joints()
+ # split_train_test() #
+
+ ## read images dir
+ logging.info("reading images list ...")
+ img_dir = os.path.join(path, extracted_filename2)
+ _img_list = load_file_list(path=os.path.join(path, extracted_filename2), regx='\\.jpg', printable=False)
+ # ann_list = json.load(open(os.path.join(path, 'data.json')))
+ for i, im in enumerate(img_train_list):
+ if im not in _img_list:
+ print('missing training image {} in {} (remove from img(ann)_train_list)'.format(im, img_dir))
+ # img_train_list.remove(im)
+ del img_train_list[i]
+ del ann_train_list[i]
+ for i, im in enumerate(img_test_list):
+ if im not in _img_list:
+ print('missing testing image {} in {} (remove from img(ann)_test_list)'.format(im, img_dir))
+ # img_test_list.remove(im)
+ del img_train_list[i]
+ del ann_train_list[i]
+
+ ## check annotation and images
+ n_train_images = len(img_train_list)
+ n_test_images = len(img_test_list)
+ n_images = n_train_images + n_test_images
+ logging.info("n_images: {} n_train_images: {} n_test_images: {}".format(n_images, n_train_images, n_test_images))
+ n_train_ann = len(ann_train_list)
+ n_test_ann = len(ann_test_list)
+ n_ann = n_train_ann + n_test_ann
+ logging.info("n_ann: {} n_train_ann: {} n_test_ann: {}".format(n_ann, n_train_ann, n_test_ann))
+ n_train_people = len(sum(ann_train_list, []))
+ n_test_people = len(sum(ann_test_list, []))
+ n_people = n_train_people + n_test_people
+ logging.info("n_people: {} n_train_people: {} n_test_people: {}".format(n_people, n_train_people, n_test_people))
+ # add path to all image file name
+ for i, value in enumerate(img_train_list):
+ img_train_list[i] = os.path.join(img_dir, value)
+ for i, value in enumerate(img_test_list):
+ img_test_list[i] = os.path.join(img_dir, value)
+ return img_train_list, ann_train_list, img_test_list, ann_test_list
diff --git a/tensorlayer/files/dataset_loaders/nietzsche_dataset.py b/tensorlayer/files/dataset_loaders/nietzsche_dataset.py
new file mode 100644
index 0000000..3cd0e27
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/nietzsche_dataset.py
@@ -0,0 +1,42 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+
+from tensorlayer import logging
+from tensorlayer.files.utils import maybe_download_and_extract
+
+__all__ = ['load_nietzsche_dataset']
+
+
+def load_nietzsche_dataset(path='data'):
+ """Load Nietzsche dataset.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/nietzsche/``.
+
+ Returns
+ --------
+ str
+ The content.
+
+ Examples
+ --------
+ >>> see tutorial_generate_text.py
+ >>> words = tl.files.load_nietzsche_dataset()
+ >>> words = basic_clean_str(words)
+ >>> words = words.split()
+
+ """
+ logging.info("Load or Download nietzsche dataset > {}".format(path))
+ path = os.path.join(path, 'nietzsche')
+
+ filename = "nietzsche.txt"
+ url = 'https://s3.amazonaws.com/text-datasets/'
+ filepath = maybe_download_and_extract(filename, path, url)
+
+ with open(filepath, "r") as f:
+ words = f.read()
+ return words
diff --git a/tensorlayer/files/dataset_loaders/ptb_dataset.py b/tensorlayer/files/dataset_loaders/ptb_dataset.py
new file mode 100644
index 0000000..30746fd
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/ptb_dataset.py
@@ -0,0 +1,72 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+
+from tensorlayer import logging, nlp
+from tensorlayer.files.utils import maybe_download_and_extract
+
+__all__ = ['load_ptb_dataset']
+
+
+def load_ptb_dataset(path='data'):
+ """Load Penn TreeBank (PTB) dataset.
+
+ It is used in many LANGUAGE MODELING papers,
+ including "Empirical Evaluation and Combination of Advanced Language
+ Modeling Techniques", "Recurrent Neural Network Regularization".
+ It consists of 929k training words, 73k validation words, and 82k test
+ words. It has 10k words in its vocabulary.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/ptb/``.
+
+ Returns
+ --------
+ train_data, valid_data, test_data : list of int
+ The training, validating and testing data in integer format.
+ vocab_size : int
+ The vocabulary size.
+
+ Examples
+ --------
+ >>> train_data, valid_data, test_data, vocab_size = tl.files.load_ptb_dataset()
+
+ References
+ ---------------
+ - ``tensorflow.models.rnn.ptb import reader``
+ - `Manual download `__
+
+ Notes
+ ------
+ - If you want to get the raw data, see the source code.
+
+ """
+ path = os.path.join(path, 'ptb')
+ logging.info("Load or Download Penn TreeBank (PTB) dataset > {}".format(path))
+
+ #Maybe dowload and uncompress tar, or load exsisting files
+ filename = 'simple-examples.tgz'
+ url = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/'
+ maybe_download_and_extract(filename, path, url, extract=True)
+
+ data_path = os.path.join(path, 'simple-examples', 'data')
+ train_path = os.path.join(data_path, "ptb.train.txt")
+ valid_path = os.path.join(data_path, "ptb.valid.txt")
+ test_path = os.path.join(data_path, "ptb.test.txt")
+
+ word_to_id = nlp.build_vocab(nlp.read_words(train_path))
+
+ train_data = nlp.words_to_word_ids(nlp.read_words(train_path), word_to_id)
+ valid_data = nlp.words_to_word_ids(nlp.read_words(valid_path), word_to_id)
+ test_data = nlp.words_to_word_ids(nlp.read_words(test_path), word_to_id)
+ vocab_size = len(word_to_id)
+
+ # logging.info(nlp.read_words(train_path)) # ... 'according', 'to', 'mr.', '', '']
+ # logging.info(train_data) # ... 214, 5, 23, 1, 2]
+ # logging.info(word_to_id) # ... 'beyond': 1295, 'anti-nuclear': 9599, 'trouble': 1520, '': 2 ... }
+ # logging.info(vocabulary) # 10000
+ # exit()
+ return train_data, valid_data, test_data, vocab_size
diff --git a/tensorlayer/files/dataset_loaders/voc_dataset.py b/tensorlayer/files/dataset_loaders/voc_dataset.py
new file mode 100644
index 0000000..5584864
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/voc_dataset.py
@@ -0,0 +1,335 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+
+import tensorflow as tf
+
+from tensorlayer import logging, utils
+from tensorlayer.files.utils import (del_file, del_folder, folder_exists, load_file_list, maybe_download_and_extract)
+
+__all__ = ['load_voc_dataset']
+
+
+def load_voc_dataset(path='data', dataset='2012', contain_classes_in_person=False):
+ """Pascal VOC 2007/2012 Dataset.
+
+ It has 20 objects:
+ aeroplane, bicycle, bird, boat, bottle, bus, car, cat, chair, cow, diningtable, dog, horse, motorbike, person, pottedplant, sheep, sofa, train, tvmonitor
+ and additional 3 classes : head, hand, foot for person.
+
+ Parameters
+ -----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/VOC``.
+ dataset : str
+ The VOC dataset version, `2012`, `2007`, `2007test` or `2012test`. We usually train model on `2007+2012` and test it on `2007test`.
+ contain_classes_in_person : boolean
+ Whether include head, hand and foot annotation, default is False.
+
+ Returns
+ ---------
+ imgs_file_list : list of str
+ Full paths of all images.
+ imgs_semseg_file_list : list of str
+ Full paths of all maps for semantic segmentation. Note that not all images have this map!
+ imgs_insseg_file_list : list of str
+ Full paths of all maps for instance segmentation. Note that not all images have this map!
+ imgs_ann_file_list : list of str
+ Full paths of all annotations for bounding box and object class, all images have this annotations.
+ classes : list of str
+ Classes in order.
+ classes_in_person : list of str
+ Classes in person.
+ classes_dict : dictionary
+ Class label to integer.
+ n_objs_list : list of int
+ Number of objects in all images in ``imgs_file_list`` in order.
+ objs_info_list : list of str
+ Darknet format for the annotation of all images in ``imgs_file_list`` in order. ``[class_id x_centre y_centre width height]`` in ratio format.
+ objs_info_dicts : dictionary
+ The annotation of all images in ``imgs_file_list``, ``{imgs_file_list : dictionary for annotation}``,
+ format from `TensorFlow/Models/object-detection `__.
+
+ Examples
+ ----------
+ >>> imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list,
+ >>> classes, classes_in_person, classes_dict,
+ >>> n_objs_list, objs_info_list, objs_info_dicts = tl.files.load_voc_dataset(dataset="2012", contain_classes_in_person=False)
+ >>> idx = 26
+ >>> print(classes)
+ ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
+ >>> print(classes_dict)
+ {'sheep': 16, 'horse': 12, 'bicycle': 1, 'bottle': 4, 'cow': 9, 'sofa': 17, 'car': 6, 'dog': 11, 'cat': 7, 'person': 14, 'train': 18, 'diningtable': 10, 'aeroplane': 0, 'bus': 5, 'pottedplant': 15, 'tvmonitor': 19, 'chair': 8, 'bird': 2, 'boat': 3, 'motorbike': 13}
+ >>> print(imgs_file_list[idx])
+ data/VOC/VOC2012/JPEGImages/2007_000423.jpg
+ >>> print(n_objs_list[idx])
+ 2
+ >>> print(imgs_ann_file_list[idx])
+ data/VOC/VOC2012/Annotations/2007_000423.xml
+ >>> print(objs_info_list[idx])
+ 14 0.173 0.461333333333 0.142 0.496
+ 14 0.828 0.542666666667 0.188 0.594666666667
+ >>> ann = tl.prepro.parse_darknet_ann_str_to_list(objs_info_list[idx])
+ >>> print(ann)
+ [[14, 0.173, 0.461333333333, 0.142, 0.496], [14, 0.828, 0.542666666667, 0.188, 0.594666666667]]
+ >>> c, b = tl.prepro.parse_darknet_ann_list_to_cls_box(ann)
+ >>> print(c, b)
+ [14, 14] [[0.173, 0.461333333333, 0.142, 0.496], [0.828, 0.542666666667, 0.188, 0.594666666667]]
+
+ References
+ -------------
+ - `Pascal VOC2012 Website `__.
+ - `Pascal VOC2007 Website `__.
+
+ """
+ try:
+ import lxml.etree as etree
+ except ImportError as e:
+ print(e)
+ raise ImportError("Module lxml not found. Please install lxml via pip or other package managers.")
+
+ path = os.path.join(path, 'VOC')
+
+ def _recursive_parse_xml_to_dict(xml):
+ """Recursively parses XML contents to python dict.
+
+ We assume that `object` tags are the only ones that can appear
+ multiple times at the same level of a tree.
+
+ Args:
+ xml: xml tree obtained by parsing XML file contents using lxml.etree
+
+ Returns:
+ Python dictionary holding XML contents.
+
+ """
+ if xml is not None:
+ return {xml.tag: xml.text}
+ result = {}
+ for child in xml:
+ child_result = _recursive_parse_xml_to_dict(child)
+ if child.tag != 'object':
+ result[child.tag] = child_result[child.tag]
+ else:
+ if child.tag not in result:
+ result[child.tag] = []
+ result[child.tag].append(child_result[child.tag])
+ return {xml.tag: result}
+
+ import xml.etree.ElementTree as ET
+
+ if dataset == "2012":
+ url = "http://pjreddie.com/media/files/"
+ tar_filename = "VOCtrainval_11-May-2012.tar"
+ extracted_filename = "VOC2012" #"VOCdevkit/VOC2012"
+ logging.info(" [============= VOC 2012 =============]")
+ elif dataset == "2012test":
+ extracted_filename = "VOC2012test" #"VOCdevkit/VOC2012"
+ logging.info(" [============= VOC 2012 Test Set =============]")
+ logging.info(
+ " \nAuthor: 2012test only have person annotation, so 2007test is highly recommended for testing !\n"
+ )
+ import time
+ time.sleep(3)
+ if os.path.isdir(os.path.join(path, extracted_filename)) is False:
+ logging.info("For VOC 2012 Test data - online registration required")
+ logging.info(
+ " Please download VOC2012test.tar from: \n register: http://host.robots.ox.ac.uk:8080 \n voc2012 : http://host.robots.ox.ac.uk:8080/eval/challenges/voc2012/ \ndownload: http://host.robots.ox.ac.uk:8080/eval/downloads/VOC2012test.tar"
+ )
+ logging.info(" unzip VOC2012test.tar,rename the folder to VOC2012test and put it into %s" % path)
+ exit()
+ # # http://host.robots.ox.ac.uk:8080/eval/downloads/VOC2012test.tar
+ # url = "http://host.robots.ox.ac.uk:8080/eval/downloads/"
+ # tar_filename = "VOC2012test.tar"
+ elif dataset == "2007":
+ url = "http://pjreddie.com/media/files/"
+ tar_filename = "VOCtrainval_06-Nov-2007.tar"
+ extracted_filename = "VOC2007"
+ logging.info(" [============= VOC 2007 =============]")
+ elif dataset == "2007test":
+ # http://host.robots.ox.ac.uk/pascal/VOC/voc2007/index.html#testdata
+ # http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar
+ url = "http://pjreddie.com/media/files/"
+ tar_filename = "VOCtest_06-Nov-2007.tar"
+ extracted_filename = "VOC2007test"
+ logging.info(" [============= VOC 2007 Test Set =============]")
+ else:
+ raise Exception("Please set the dataset aug to 2012, 2012test or 2007.")
+
+ # download dataset
+ if dataset != "2012test":
+ from sys import platform as _platform
+ if folder_exists(os.path.join(path, extracted_filename)) is False:
+ logging.info("[VOC] {} is nonexistent in {}".format(extracted_filename, path))
+ maybe_download_and_extract(tar_filename, path, url, extract=True)
+ del_file(os.path.join(path, tar_filename))
+ if dataset == "2012":
+ if _platform == "win32":
+ os.system("move {}\VOCdevkit\VOC2012 {}\VOC2012".format(path, path))
+ else:
+ os.system("mv {}/VOCdevkit/VOC2012 {}/VOC2012".format(path, path))
+ elif dataset == "2007":
+ if _platform == "win32":
+ os.system("move {}\VOCdevkit\VOC2007 {}\VOC2007".format(path, path))
+ else:
+ os.system("mv {}/VOCdevkit/VOC2007 {}/VOC2007".format(path, path))
+ elif dataset == "2007test":
+ if _platform == "win32":
+ os.system("move {}\VOCdevkit\VOC2007 {}\VOC2007test".format(path, path))
+ else:
+ os.system("mv {}/VOCdevkit/VOC2007 {}/VOC2007test".format(path, path))
+ del_folder(os.path.join(path, 'VOCdevkit'))
+ # object classes(labels) NOTE: YOU CAN CUSTOMIZE THIS LIST
+ classes = [
+ "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog",
+ "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"
+ ]
+ if contain_classes_in_person:
+ classes_in_person = ["head", "hand", "foot"]
+ else:
+ classes_in_person = []
+
+ classes += classes_in_person # use extra 3 classes for person
+
+ classes_dict = utils.list_string_to_dict(classes)
+ logging.info("[VOC] object classes {}".format(classes_dict))
+
+ # 1. image path list
+ # folder_imgs = path+"/"+extracted_filename+"/JPEGImages/"
+ folder_imgs = os.path.join(path, extracted_filename, "JPEGImages")
+ imgs_file_list = load_file_list(path=folder_imgs, regx='\\.jpg', printable=False)
+ logging.info("[VOC] {} images found".format(len(imgs_file_list)))
+
+ imgs_file_list.sort(
+ key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])
+ ) # 2007_000027.jpg --> 2007000027
+
+ imgs_file_list = [os.path.join(folder_imgs, s) for s in imgs_file_list]
+ # logging.info('IM',imgs_file_list[0::3333], imgs_file_list[-1])
+ if dataset != "2012test":
+ ##======== 2. semantic segmentation maps path list
+ # folder_semseg = path+"/"+extracted_filename+"/SegmentationClass/"
+ folder_semseg = os.path.join(path, extracted_filename, "SegmentationClass")
+ imgs_semseg_file_list = load_file_list(path=folder_semseg, regx='\\.png', printable=False)
+ logging.info("[VOC] {} maps for semantic segmentation found".format(len(imgs_semseg_file_list)))
+ imgs_semseg_file_list.sort(
+ key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])
+ ) # 2007_000032.png --> 2007000032
+ imgs_semseg_file_list = [os.path.join(folder_semseg, s) for s in imgs_semseg_file_list]
+ # logging.info('Semantic Seg IM',imgs_semseg_file_list[0::333], imgs_semseg_file_list[-1])
+ ##======== 3. instance segmentation maps path list
+ # folder_insseg = path+"/"+extracted_filename+"/SegmentationObject/"
+ folder_insseg = os.path.join(path, extracted_filename, "SegmentationObject")
+ imgs_insseg_file_list = load_file_list(path=folder_insseg, regx='\\.png', printable=False)
+ logging.info("[VOC] {} maps for instance segmentation found".format(len(imgs_semseg_file_list)))
+ imgs_insseg_file_list.sort(
+ key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])
+ ) # 2007_000032.png --> 2007000032
+ imgs_insseg_file_list = [os.path.join(folder_insseg, s) for s in imgs_insseg_file_list]
+ # logging.info('Instance Seg IM',imgs_insseg_file_list[0::333], imgs_insseg_file_list[-1])
+ else:
+ imgs_semseg_file_list = []
+ imgs_insseg_file_list = []
+ # 4. annotations for bounding box and object class
+ # folder_ann = path+"/"+extracted_filename+"/Annotations/"
+ folder_ann = os.path.join(path, extracted_filename, "Annotations")
+ imgs_ann_file_list = load_file_list(path=folder_ann, regx='\\.xml', printable=False)
+ logging.info(
+ "[VOC] {} XML annotation files for bounding box and object class found".format(len(imgs_ann_file_list))
+ )
+ imgs_ann_file_list.sort(
+ key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])
+ ) # 2007_000027.xml --> 2007000027
+ imgs_ann_file_list = [os.path.join(folder_ann, s) for s in imgs_ann_file_list]
+ # logging.info('ANN',imgs_ann_file_list[0::3333], imgs_ann_file_list[-1])
+
+ if dataset == "2012test": # remove unused images in JPEG folder
+ imgs_file_list_new = []
+ for ann in imgs_ann_file_list:
+ ann = os.path.split(ann)[-1].split('.')[0]
+ for im in imgs_file_list:
+ if ann in im:
+ imgs_file_list_new.append(im)
+ break
+ imgs_file_list = imgs_file_list_new
+ logging.info("[VOC] keep %d images" % len(imgs_file_list_new))
+
+ # parse XML annotations
+ def convert(size, box):
+ dw = 1. / size[0]
+ dh = 1. / size[1]
+ x = (box[0] + box[1]) / 2.0
+ y = (box[2] + box[3]) / 2.0
+ w = box[1] - box[0]
+ h = box[3] - box[2]
+ x = x * dw
+ w = w * dw
+ y = y * dh
+ h = h * dh
+ return x, y, w, h
+
+ def convert_annotation(file_name):
+ """Given VOC2012 XML Annotations, returns number of objects and info."""
+ in_file = open(file_name)
+ out_file = ""
+ tree = ET.parse(in_file)
+ root = tree.getroot()
+ size = root.find('size')
+ w = int(size.find('width').text)
+ h = int(size.find('height').text)
+ n_objs = 0
+
+ for obj in root.iter('object'):
+ if dataset != "2012test":
+ difficult = obj.find('difficult').text
+ cls = obj.find('name').text
+ if cls not in classes or int(difficult) == 1:
+ continue
+ else:
+ cls = obj.find('name').text
+ if cls not in classes:
+ continue
+ cls_id = classes.index(cls)
+ xmlbox = obj.find('bndbox')
+ b = (
+ float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),
+ float(xmlbox.find('ymax').text)
+ )
+ bb = convert((w, h), b)
+
+ out_file += str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n'
+ n_objs += 1
+ if cls in "person":
+ for part in obj.iter('part'):
+ cls = part.find('name').text
+ if cls not in classes_in_person:
+ continue
+ cls_id = classes.index(cls)
+ xmlbox = part.find('bndbox')
+ b = (
+ float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text),
+ float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text)
+ )
+ bb = convert((w, h), b)
+ # out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
+ out_file += str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n'
+ n_objs += 1
+ in_file.close()
+ return n_objs, out_file
+
+ logging.info("[VOC] Parsing xml annotations files")
+ n_objs_list = []
+ objs_info_list = [] # Darknet Format list of string
+ objs_info_dicts = {}
+ for idx, ann_file in enumerate(imgs_ann_file_list):
+ n_objs, objs_info = convert_annotation(ann_file)
+ n_objs_list.append(n_objs)
+ objs_info_list.append(objs_info)
+ with tf.io.gfile.GFile(ann_file, 'r') as fid:
+ xml_str = fid.read()
+ xml = etree.fromstring(xml_str)
+ data = _recursive_parse_xml_to_dict(xml)['annotation']
+ objs_info_dicts.update({imgs_file_list[idx]: data})
+
+ return imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list, classes, classes_in_person, classes_dict, n_objs_list, objs_info_list, objs_info_dicts
diff --git a/tensorlayer/files/dataset_loaders/wmt_en_fr_dataset.py b/tensorlayer/files/dataset_loaders/wmt_en_fr_dataset.py
new file mode 100644
index 0000000..0261a85
--- /dev/null
+++ b/tensorlayer/files/dataset_loaders/wmt_en_fr_dataset.py
@@ -0,0 +1,80 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import gzip
+import os
+import tarfile
+
+from tensorflow.python.platform import gfile
+
+from tensorlayer import logging
+from tensorlayer.files.utils import maybe_download_and_extract
+
+__all__ = ['load_wmt_en_fr_dataset']
+
+
+def load_wmt_en_fr_dataset(path='data'):
+ """Load WMT'15 English-to-French translation dataset.
+
+ It will download the data from the WMT'15 Website (10^9-French-English corpus), and the 2013 news test from the same site as development set.
+ Returns the directories of training data and test data.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/wmt_en_fr/``.
+
+ References
+ ----------
+ - Code modified from /tensorflow/models/rnn/translation/data_utils.py
+
+ Notes
+ -----
+ Usually, it will take a long time to download this dataset.
+
+ """
+ path = os.path.join(path, 'wmt_en_fr')
+ # URLs for WMT data.
+ _WMT_ENFR_TRAIN_URL = "http://www.statmt.org/wmt10/"
+ _WMT_ENFR_DEV_URL = "http://www.statmt.org/wmt15/"
+
+ def gunzip_file(gz_path, new_path):
+ """Unzips from gz_path into new_path."""
+ logging.info("Unpacking %s to %s" % (gz_path, new_path))
+ with gzip.open(gz_path, "rb") as gz_file:
+ with open(new_path, "wb") as new_file:
+ for line in gz_file:
+ new_file.write(line)
+
+ def get_wmt_enfr_train_set(path):
+ """Download the WMT en-fr training corpus to directory unless it's there."""
+ filename = "training-giga-fren.tar"
+ maybe_download_and_extract(filename, path, _WMT_ENFR_TRAIN_URL, extract=True)
+ train_path = os.path.join(path, "giga-fren.release2.fixed")
+ gunzip_file(train_path + ".fr.gz", train_path + ".fr")
+ gunzip_file(train_path + ".en.gz", train_path + ".en")
+ return train_path
+
+ def get_wmt_enfr_dev_set(path):
+ """Download the WMT en-fr training corpus to directory unless it's there."""
+ filename = "dev-v2.tgz"
+ dev_file = maybe_download_and_extract(filename, path, _WMT_ENFR_DEV_URL, extract=False)
+ dev_name = "newstest2013"
+ dev_path = os.path.join(path, "newstest2013")
+ if not (gfile.Exists(dev_path + ".fr") and gfile.Exists(dev_path + ".en")):
+ logging.info("Extracting tgz file %s" % dev_file)
+ with tarfile.open(dev_file, "r:gz") as dev_tar:
+ fr_dev_file = dev_tar.getmember("dev/" + dev_name + ".fr")
+ en_dev_file = dev_tar.getmember("dev/" + dev_name + ".en")
+ fr_dev_file.name = dev_name + ".fr" # Extract without "dev/" prefix.
+ en_dev_file.name = dev_name + ".en"
+ dev_tar.extract(fr_dev_file, path)
+ dev_tar.extract(en_dev_file, path)
+ return dev_path
+
+ logging.info("Load or Download WMT English-to-French translation > {}".format(path))
+
+ train_path = get_wmt_enfr_train_set(path)
+ dev_path = get_wmt_enfr_dev_set(path)
+
+ return train_path, dev_path
diff --git a/tensorlayer/files/utils.py b/tensorlayer/files/utils.py
new file mode 100644
index 0000000..d05a0c3
--- /dev/null
+++ b/tensorlayer/files/utils.py
@@ -0,0 +1,2932 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import base64
+import datetime
+import gzip
+import json
+import math
+import os
+import pickle
+import re
+import shutil
+# import ast
+import sys
+import tarfile
+import time
+import zipfile
+
+import cloudpickle
+import h5py
+import numpy as np
+import progressbar
+import scipy.io as sio
+import tensorflow as tf
+from six.moves import cPickle
+from tensorflow.python.keras.saving import model_config as model_config_lib
+from tensorflow.python.platform import gfile
+from tensorflow.python.util import serialization
+from tensorflow.python.util.tf_export import keras_export
+from tensorflow.python import pywrap_tensorflow
+
+import tensorlayer as tl
+from tensorlayer import logging, nlp, utils, visualize
+
+if tl.BACKEND == 'mindspore':
+ from mindspore.ops.operations import Assign
+ from mindspore.nn import Cell
+ from mindspore import Tensor
+ import mindspore as ms
+
+if sys.version_info[0] == 2:
+ from urllib import urlretrieve
+else:
+ from urllib.request import urlretrieve
+
+# import tensorflow.contrib.eager.python.saver as tfes
+# TODO: tf2.0 not stable, cannot import tensorflow.contrib.eager.python.saver
+
+__all__ = [
+ 'assign_weights',
+ 'del_file',
+ 'del_folder',
+ 'download_file_from_google_drive',
+ 'exists_or_mkdir',
+ 'file_exists',
+ 'folder_exists',
+ 'load_and_assign_npz',
+ 'load_and_assign_npz_dict',
+ 'load_ckpt',
+ 'load_cropped_svhn',
+ 'load_file_list',
+ 'load_folder_list',
+ 'load_npy_to_any',
+ 'load_npz',
+ 'maybe_download_and_extract',
+ 'natural_keys',
+ 'npz_to_W_pdf',
+ 'read_file',
+ 'save_any_to_npy',
+ 'save_ckpt',
+ 'save_npz',
+ 'save_npz_dict',
+ 'tf_variables_to_numpy',
+ 'ms_variables_to_numpy',
+ 'assign_tf_variable',
+ 'assign_ms_variable',
+ 'save_weights_to_hdf5',
+ 'load_hdf5_to_weights_in_order',
+ 'load_hdf5_to_weights',
+ 'save_hdf5_graph',
+ 'load_hdf5_graph',
+ # 'net2static_graph',
+ 'static_graph2net',
+ # 'save_pkl_graph',
+ # 'load_pkl_graph',
+ 'load_and_assign_ckpt',
+ 'ckpt_to_npz_dict'
+]
+
+
+def func2str(expr):
+ b = cloudpickle.dumps(expr)
+ s = base64.b64encode(b).decode()
+ return s
+
+
+def str2func(s):
+ b = base64.b64decode(s)
+ expr = cloudpickle.loads(b)
+ return expr
+
+
+# def net2static_graph(network):
+# saved_file = dict()
+# # if network._NameNone is True:
+# # saved_file.update({"name": None})
+# # else:
+# # saved_file.update({"name": network.name})
+# # if not isinstance(network.inputs, list):
+# # saved_file.update({"inputs": network.inputs._info[0].name})
+# # else:
+# # saved_inputs = []
+# # for saved_input in network.inputs:
+# # saved_inputs.append(saved_input._info[0].name)
+# # saved_file.update({"inputs": saved_inputs})
+# # if not isinstance(network.outputs, list):
+# # saved_file.update({"outputs": network.outputs._info[0].name})
+# # else:
+# # saved_outputs = []
+# # for saved_output in network.outputs:
+# # saved_outputs.append(saved_output._info[0].name)
+# # saved_file.update({"outputs": saved_outputs})
+# saved_file.update({"config": network.config})
+#
+# return saved_file
+
+
+@keras_export('keras.models.save_model')
+def save_keras_model(model):
+ # f.attrs['keras_model_config'] = json.dumps(
+ # {
+ # 'class_name': model.__class__.__name__,
+ # 'config': model.get_config()
+ # },
+ # default=serialization.get_json_type).encode('utf8')
+ #
+ # f.flush()
+
+ return json.dumps(
+ {
+ 'class_name': model.__class__.__name__,
+ 'config': model.get_config()
+ }, default=serialization.get_json_type
+ ).encode('utf8')
+
+
+@keras_export('keras.models.load_model')
+def load_keras_model(model_config):
+
+ custom_objects = {}
+
+ if model_config is None:
+ raise ValueError('No model found in config.')
+ model_config = json.loads(model_config.decode('utf-8'))
+ model = model_config_lib.model_from_config(model_config, custom_objects=custom_objects)
+
+ return model
+
+
+def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False, customized_data=None):
+ """Save the architecture of TL model into a hdf5 file. Support saving model weights.
+
+ Parameters
+ -----------
+ network : TensorLayer Model.
+ The network to save.
+ filepath : str
+ The name of model file.
+ save_weights : bool
+ Whether to save model weights.
+ customized_data : dict
+ The user customized meta data.
+
+ Examples
+ --------
+ >>> # Save the architecture (with parameters)
+ >>> tl.files.save_hdf5_graph(network, filepath='model.hdf5', save_weights=True)
+ >>> # Save the architecture (without parameters)
+ >>> tl.files.save_hdf5_graph(network, filepath='model.hdf5', save_weights=False)
+ >>> # Load the architecture in another script (no parameters restore)
+ >>> net = tl.files.load_hdf5_graph(filepath='model.hdf5', load_weights=False)
+ >>> # Load the architecture in another script (restore parameters)
+ >>> net = tl.files.load_hdf5_graph(filepath='model.hdf5', load_weights=True)
+ """
+ if network.outputs is None:
+ raise RuntimeError("save_hdf5_graph not support dynamic mode yet")
+
+ logging.info("[*] Saving TL model into {}, saving weights={}".format(filepath, save_weights))
+
+ model_config = network.config # net2static_graph(network)
+ model_config["version_info"]["save_date"] = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc
+ ).isoformat()
+ model_config_str = str(model_config)
+ customized_data_str = str(customized_data)
+ # version_info = {
+ # "tensorlayer_version": tl.__version__,
+ # "backend": "tensorflow",
+ # "backend_version": tf.__version__,
+ # "training_device": "gpu",
+ # "save_date": datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat()
+ # }
+ # version_info_str = str(version_info)
+
+ with h5py.File(filepath, 'w') as f:
+ f.attrs["model_config"] = model_config_str.encode('utf8')
+ f.attrs["customized_data"] = customized_data_str.encode('utf8')
+ # f.attrs["version_info"] = version_info_str.encode('utf8')
+ if save_weights:
+ _save_weights_to_hdf5_group(f, network.all_layers)
+ f.flush()
+
+ logging.info("[*] Saved TL model into {}, saving weights={}".format(filepath, save_weights))
+
+
+def generate_func(args):
+ for key in args:
+ if isinstance(args[key], tuple) and args[key][0] == 'is_Func':
+ fn = str2func(args[key][1])
+ args[key] = fn
+ # if key in ['act']:
+ # # fn_dict = args[key]
+ # # module_path = fn_dict['module_path']
+ # # func_name = fn_dict['func_name']
+ # # lib = importlib.import_module(module_path)
+ # # fn = getattr(lib, func_name)
+ # # args[key] = fn
+ # fn = str2func(args[key])
+ # args[key] = fn
+ # elif key in ['fn']:
+ # fn = str2func(args[key])
+ # args[key] = fn
+
+
+def eval_layer(layer_kwargs):
+ layer_class = layer_kwargs.pop('class')
+ args = layer_kwargs['args']
+ layer_type = args.pop('layer_type')
+ if layer_type == "normal":
+ generate_func(args)
+ return eval('tl.layers.' + layer_class)(**args)
+ elif layer_type == "layerlist":
+ ret_layer = []
+ layers = args["layers"]
+ for layer_graph in layers:
+ ret_layer.append(eval_layer(layer_graph))
+ args['layers'] = ret_layer
+ return eval('tl.layers.' + layer_class)(**args)
+ elif layer_type == "modellayer":
+ M = static_graph2net(args['model'])
+ args['model'] = M
+ return eval('tl.layers.' + layer_class)(**args)
+ elif layer_type == "keraslayer":
+ M = load_keras_model(args['fn'])
+ input_shape = args.pop('keras_input_shape')
+ _ = M(np.random.random(input_shape).astype(np.float32))
+ args['fn'] = M
+ args['fn_weights'] = M.trainable_variables
+ return eval('tl.layers.' + layer_class)(**args)
+ else:
+ raise RuntimeError("Unknown layer type.")
+
+
+def static_graph2net(model_config):
+ layer_dict = {}
+ model_name = model_config["name"]
+ inputs_tensors = model_config["inputs"]
+ outputs_tensors = model_config["outputs"]
+ all_args = model_config["model_architecture"]
+ for idx, layer_kwargs in enumerate(all_args):
+ layer_class = layer_kwargs["class"] # class of current layer
+ prev_layers = layer_kwargs.pop("prev_layer") # name of previous layers
+ net = eval_layer(layer_kwargs)
+ if layer_class in tl.layers.inputs.__all__:
+ net = net._nodes[0].out_tensors[0]
+ if prev_layers is not None:
+ for prev_layer in prev_layers:
+ if not isinstance(prev_layer, list):
+ output = net(layer_dict[prev_layer])
+ layer_dict[output._info[0].name] = output
+ else:
+ list_layers = [layer_dict[layer] for layer in prev_layer]
+ output = net(list_layers)
+ layer_dict[output._info[0].name] = output
+ else:
+ layer_dict[net._info[0].name] = net
+
+ if not isinstance(inputs_tensors, list):
+ model_inputs = layer_dict[inputs_tensors]
+ else:
+ model_inputs = []
+ for inputs_tensor in inputs_tensors:
+ model_inputs.append(layer_dict[inputs_tensor])
+ if not isinstance(outputs_tensors, list):
+ model_outputs = layer_dict[outputs_tensors]
+ else:
+ model_outputs = []
+ for outputs_tensor in outputs_tensors:
+ model_outputs.append(layer_dict[outputs_tensor])
+ from tensorlayer.models import Model
+ M = Model(inputs=model_inputs, outputs=model_outputs, name=model_name)
+ logging.info("[*] Load graph finished")
+ return M
+
+
+def load_hdf5_graph(filepath='model.hdf5', load_weights=False):
+ """Restore TL model archtecture from a a pickle file. Support loading model weights.
+
+ Parameters
+ -----------
+ filepath : str
+ The name of model file.
+ load_weights : bool
+ Whether to load model weights.
+
+ Returns
+ --------
+ network : TensorLayer Model.
+
+ Examples
+ --------
+ - see ``tl.files.save_hdf5_graph``
+ """
+ logging.info("[*] Loading TL model from {}, loading weights={}".format(filepath, load_weights))
+
+ f = h5py.File(filepath, 'r')
+
+ model_config_str = f.attrs["model_config"].decode('utf8')
+ model_config = eval(model_config_str)
+
+ # version_info_str = f.attrs["version_info"].decode('utf8')
+ # version_info = eval(version_info_str)
+ version_info = model_config["version_info"]
+ backend_version = version_info["backend_version"]
+ tensorlayer_version = version_info["tensorlayer_version"]
+ if backend_version != tf.__version__:
+ logging.warning(
+ "Saved model uses tensorflow version {}, but now you are using tensorflow version {}".format(
+ backend_version, tf.__version__
+ )
+ )
+ if tensorlayer_version != tl.__version__:
+ logging.warning(
+ "Saved model uses tensorlayer version {}, but now you are using tensorlayer version {}".format(
+ tensorlayer_version, tl.__version__
+ )
+ )
+
+ M = static_graph2net(model_config)
+ if load_weights:
+ if not ('layer_names' in f.attrs.keys()):
+ raise RuntimeError("Saved model does not contain weights.")
+ M.load_weights(filepath=filepath)
+
+ f.close()
+
+ logging.info("[*] Loaded TL model from {}, loading weights={}".format(filepath, load_weights))
+
+ return M
+
+
+# def load_pkl_graph(name='model.pkl'):
+# """Restore TL model archtecture from a a pickle file. No parameters be restored.
+#
+# Parameters
+# -----------
+# name : str
+# The name of graph file.
+#
+# Returns
+# --------
+# network : TensorLayer Model.
+#
+# Examples
+# --------
+# >>> # It is better to use load_hdf5_graph
+# """
+# logging.info("[*] Loading TL graph from {}".format(name))
+# with open(name, 'rb') as file:
+# saved_file = pickle.load(file)
+#
+# M = static_graph2net(saved_file)
+#
+# return M
+#
+#
+# def save_pkl_graph(network, name='model.pkl'):
+# """Save the architecture of TL model into a pickle file. No parameters be saved.
+#
+# Parameters
+# -----------
+# network : TensorLayer layer
+# The network to save.
+# name : str
+# The name of graph file.
+#
+# Example
+# --------
+# >>> # It is better to use save_hdf5_graph
+# """
+# if network.outputs is None:
+# raise AssertionError("save_graph not support dynamic mode yet")
+#
+# logging.info("[*] Saving TL graph into {}".format(name))
+#
+# saved_file = net2static_graph(network)
+#
+# with open(name, 'wb') as file:
+# pickle.dump(saved_file, file, protocol=pickle.HIGHEST_PROTOCOL)
+# logging.info("[*] Saved graph")
+
+
+# Load dataset functions
+def load_mnist_dataset(shape=(-1, 784), path='data'):
+ """Load the original mnist.
+
+ Automatically download MNIST dataset and return the training, validation and test set with 50000, 10000 and 10000 digit images respectively.
+
+ Parameters
+ ----------
+ shape : tuple
+ The shape of digit images (the default is (-1, 784), alternatively (-1, 28, 28, 1)).
+ path : str
+ The path that the data is downloaded to.
+
+ Returns
+ -------
+ X_train, y_train, X_val, y_val, X_test, y_test: tuple
+ Return splitted training/validation/test set respectively.
+
+ Examples
+ --------
+ >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1,784), path='datasets')
+ >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 28, 28, 1))
+ """
+ return _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/')
+
+
+def load_fashion_mnist_dataset(shape=(-1, 784), path='data'):
+ """Load the fashion mnist.
+
+ Automatically download fashion-MNIST dataset and return the training, validation and test set with 50000, 10000 and 10000 fashion images respectively, `examples `__.
+
+ Parameters
+ ----------
+ shape : tuple
+ The shape of digit images (the default is (-1, 784), alternatively (-1, 28, 28, 1)).
+ path : str
+ The path that the data is downloaded to.
+
+ Returns
+ -------
+ X_train, y_train, X_val, y_val, X_test, y_test: tuple
+ Return splitted training/validation/test set respectively.
+
+ Examples
+ --------
+ >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_fashion_mnist_dataset(shape=(-1,784), path='datasets')
+ >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_fashion_mnist_dataset(shape=(-1, 28, 28, 1))
+ """
+ return _load_mnist_dataset(
+ shape, path, name='fashion_mnist', url='http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/'
+ )
+
+
+def _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/'):
+ """A generic function to load mnist-like dataset.
+
+ Parameters:
+ ----------
+ shape : tuple
+ The shape of digit images.
+ path : str
+ The path that the data is downloaded to.
+ name : str
+ The dataset name you want to use(the default is 'mnist').
+ url : str
+ The url of dataset(the default is 'http://yann.lecun.com/exdb/mnist/').
+ """
+ path = os.path.join(path, name)
+
+ # Define functions for loading mnist-like data's images and labels.
+ # For convenience, they also download the requested files if needed.
+ def load_mnist_images(path, filename):
+ filepath = maybe_download_and_extract(filename, path, url)
+
+ logging.info(filepath)
+ # Read the inputs in Yann LeCun's binary format.
+ with gzip.open(filepath, 'rb') as f:
+ data = np.frombuffer(f.read(), np.uint8, offset=16)
+ # The inputs are vectors now, we reshape them to monochrome 2D images,
+ # following the shape convention: (examples, channels, rows, columns)
+ data = data.reshape(shape)
+ # The inputs come as bytes, we convert them to float32 in range [0,1].
+ # (Actually to range [0, 255/256], for compatibility to the version
+ # provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)
+ return data / np.float32(256)
+
+ def load_mnist_labels(path, filename):
+ filepath = maybe_download_and_extract(filename, path, url)
+ # Read the labels in Yann LeCun's binary format.
+ with gzip.open(filepath, 'rb') as f:
+ data = np.frombuffer(f.read(), np.uint8, offset=8)
+ # The labels are vectors of integers now, that's exactly what we want.
+ return data
+
+ # Download and read the training and test set images and labels.
+ logging.info("Load or Download {0} > {1}".format(name.upper(), path))
+ X_train = load_mnist_images(path, 'train-images-idx3-ubyte.gz')
+ y_train = load_mnist_labels(path, 'train-labels-idx1-ubyte.gz')
+ X_test = load_mnist_images(path, 't10k-images-idx3-ubyte.gz')
+ y_test = load_mnist_labels(path, 't10k-labels-idx1-ubyte.gz')
+
+ # We reserve the last 10000 training examples for validation.
+ X_train, X_val = X_train[:-10000], X_train[-10000:]
+ y_train, y_val = y_train[:-10000], y_train[-10000:]
+
+ # We just return all the arrays in order, as expected in main().
+ # (It doesn't matter how we do this as long as we can read them again.)
+ X_train = np.asarray(X_train, dtype=np.float32)
+ y_train = np.asarray(y_train, dtype=np.int32)
+ X_val = np.asarray(X_val, dtype=np.float32)
+ y_val = np.asarray(y_val, dtype=np.int32)
+ X_test = np.asarray(X_test, dtype=np.float32)
+ y_test = np.asarray(y_test, dtype=np.int32)
+ return X_train, y_train, X_val, y_val, X_test, y_test
+
+
+def load_cifar10_dataset(shape=(-1, 32, 32, 3), path='data', plotable=False):
+ """Load CIFAR-10 dataset.
+
+ It consists of 60000 32x32 colour images in 10 classes, with
+ 6000 images per class. There are 50000 training images and 10000 test images.
+
+ The dataset is divided into five training batches and one test batch, each with
+ 10000 images. The test batch contains exactly 1000 randomly-selected images from
+ each class. The training batches contain the remaining images in random order,
+ but some training batches may contain more images from one class than another.
+ Between them, the training batches contain exactly 5000 images from each class.
+
+ Parameters
+ ----------
+ shape : tupe
+ The shape of digit images e.g. (-1, 3, 32, 32) and (-1, 32, 32, 3).
+ path : str
+ The path that the data is downloaded to, defaults is ``data/cifar10/``.
+ plotable : boolean
+ Whether to plot some image examples, False as default.
+
+ Examples
+ --------
+ >>> X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))
+
+ References
+ ----------
+ - `CIFAR website `__
+ - `Data download link `__
+ - ``__
+
+ """
+ path = os.path.join(path, 'cifar10')
+ logging.info("Load or Download cifar10 > {}".format(path))
+
+ # Helper function to unpickle the data
+ def unpickle(file):
+ fp = open(file, 'rb')
+ if sys.version_info.major == 2:
+ data = pickle.load(fp)
+ elif sys.version_info.major == 3:
+ data = pickle.load(fp, encoding='latin-1')
+ fp.close()
+ return data
+
+ filename = 'cifar-10-python.tar.gz'
+ url = 'https://www.cs.toronto.edu/~kriz/'
+ # Download and uncompress file
+ maybe_download_and_extract(filename, path, url, extract=True)
+
+ # Unpickle file and fill in data
+ X_train = None
+ y_train = []
+ for i in range(1, 6):
+ data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', "data_batch_{}".format(i)))
+ if i == 1:
+ X_train = data_dic['data']
+ else:
+ X_train = np.vstack((X_train, data_dic['data']))
+ y_train += data_dic['labels']
+
+ test_data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', "test_batch"))
+ X_test = test_data_dic['data']
+ y_test = np.array(test_data_dic['labels'])
+
+ if shape == (-1, 3, 32, 32):
+ X_test = X_test.reshape(shape)
+ X_train = X_train.reshape(shape)
+ elif shape == (-1, 32, 32, 3):
+ X_test = X_test.reshape(shape, order='F')
+ X_train = X_train.reshape(shape, order='F')
+ X_test = np.transpose(X_test, (0, 2, 1, 3))
+ X_train = np.transpose(X_train, (0, 2, 1, 3))
+ else:
+ X_test = X_test.reshape(shape)
+ X_train = X_train.reshape(shape)
+
+ y_train = np.array(y_train)
+
+ if plotable:
+
+ if sys.platform.startswith('darwin'):
+ import matplotlib
+ matplotlib.use('TkAgg')
+ import matplotlib.pyplot as plt
+
+ logging.info('\nCIFAR-10')
+ fig = plt.figure(1)
+
+ logging.info('Shape of a training image: X_train[0] %s' % X_train[0].shape)
+
+ plt.ion() # interactive mode
+ count = 1
+ for _ in range(10): # each row
+ for _ in range(10): # each column
+ _ = fig.add_subplot(10, 10, count)
+ if shape == (-1, 3, 32, 32):
+ # plt.imshow(X_train[count-1], interpolation='nearest')
+ plt.imshow(np.transpose(X_train[count - 1], (1, 2, 0)), interpolation='nearest')
+ # plt.imshow(np.transpose(X_train[count-1], (2, 1, 0)), interpolation='nearest')
+ elif shape == (-1, 32, 32, 3):
+ plt.imshow(X_train[count - 1], interpolation='nearest')
+ # plt.imshow(np.transpose(X_train[count-1], (1, 0, 2)), interpolation='nearest')
+ else:
+ raise Exception("Do not support the given 'shape' to plot the image examples")
+ plt.gca().xaxis.set_major_locator(plt.NullLocator()) # 不显示刻度(tick)
+ plt.gca().yaxis.set_major_locator(plt.NullLocator())
+ count = count + 1
+ plt.draw() # interactive mode
+ plt.pause(3) # interactive mode
+
+ logging.info("X_train: %s" % X_train.shape)
+ logging.info("y_train: %s" % y_train.shape)
+ logging.info("X_test: %s" % X_test.shape)
+ logging.info("y_test: %s" % y_test.shape)
+
+ X_train = np.asarray(X_train, dtype=np.float32)
+ X_test = np.asarray(X_test, dtype=np.float32)
+ y_train = np.asarray(y_train, dtype=np.int32)
+ y_test = np.asarray(y_test, dtype=np.int32)
+
+ return X_train, y_train, X_test, y_test
+
+
+def load_cropped_svhn(path='data', include_extra=True):
+ """Load Cropped SVHN.
+
+ The Cropped Street View House Numbers (SVHN) Dataset contains 32x32x3 RGB images.
+ Digit '1' has label 1, '9' has label 9 and '0' has label 0 (the original dataset uses 10 to represent '0'), see `ufldl website `__.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to.
+ include_extra : boolean
+ If True (default), add extra images to the training set.
+
+ Returns
+ -------
+ X_train, y_train, X_test, y_test: tuple
+ Return splitted training/test set respectively.
+
+ Examples
+ ---------
+ >>> X_train, y_train, X_test, y_test = tl.files.load_cropped_svhn(include_extra=False)
+ >>> tl.vis.save_images(X_train[0:100], [10, 10], 'svhn.png')
+
+ """
+ start_time = time.time()
+
+ path = os.path.join(path, 'cropped_svhn')
+ logging.info("Load or Download Cropped SVHN > {} | include extra images: {}".format(path, include_extra))
+ url = "http://ufldl.stanford.edu/housenumbers/"
+
+ np_file = os.path.join(path, "train_32x32.npz")
+ if file_exists(np_file) is False:
+ filename = "train_32x32.mat"
+ filepath = maybe_download_and_extract(filename, path, url)
+ mat = sio.loadmat(filepath)
+ X_train = mat['X'] / 255.0 # to [0, 1]
+ X_train = np.transpose(X_train, (3, 0, 1, 2))
+ y_train = np.squeeze(mat['y'], axis=1)
+ y_train[y_train == 10] = 0 # replace 10 to 0
+ np.savez(np_file, X=X_train, y=y_train)
+ del_file(filepath)
+ else:
+ v = np.load(np_file, allow_pickle=True)
+ X_train = v['X']
+ y_train = v['y']
+ logging.info(" n_train: {}".format(len(y_train)))
+
+ np_file = os.path.join(path, "test_32x32.npz")
+ if file_exists(np_file) is False:
+ filename = "test_32x32.mat"
+ filepath = maybe_download_and_extract(filename, path, url)
+ mat = sio.loadmat(filepath)
+ X_test = mat['X'] / 255.0
+ X_test = np.transpose(X_test, (3, 0, 1, 2))
+ y_test = np.squeeze(mat['y'], axis=1)
+ y_test[y_test == 10] = 0
+ np.savez(np_file, X=X_test, y=y_test)
+ del_file(filepath)
+ else:
+ v = np.load(np_file, allow_pickle=True)
+ X_test = v['X']
+ y_test = v['y']
+ logging.info(" n_test: {}".format(len(y_test)))
+
+ if include_extra:
+ logging.info(" getting extra 531131 images, please wait ...")
+ np_file = os.path.join(path, "extra_32x32.npz")
+ if file_exists(np_file) is False:
+ logging.info(" the first time to load extra images will take long time to convert the file format ...")
+ filename = "extra_32x32.mat"
+ filepath = maybe_download_and_extract(filename, path, url)
+ mat = sio.loadmat(filepath)
+ X_extra = mat['X'] / 255.0
+ X_extra = np.transpose(X_extra, (3, 0, 1, 2))
+ y_extra = np.squeeze(mat['y'], axis=1)
+ y_extra[y_extra == 10] = 0
+ np.savez(np_file, X=X_extra, y=y_extra)
+ del_file(filepath)
+ else:
+ v = np.load(np_file, allow_pickle=True)
+ X_extra = v['X']
+ y_extra = v['y']
+ # print(X_train.shape, X_extra.shape)
+ logging.info(" adding n_extra {} to n_train {}".format(len(y_extra), len(y_train)))
+ t = time.time()
+ X_train = np.concatenate((X_train, X_extra), 0)
+ y_train = np.concatenate((y_train, y_extra), 0)
+ # X_train = np.append(X_train, X_extra, axis=0)
+ # y_train = np.append(y_train, y_extra, axis=0)
+ logging.info(" added n_extra {} to n_train {} took {}s".format(len(y_extra), len(y_train), time.time() - t))
+ else:
+ logging.info(" no extra images are included")
+ logging.info(" image size: %s n_train: %d n_test: %d" % (str(X_train.shape[1:4]), len(y_train), len(y_test)))
+ logging.info(" took: {}s".format(int(time.time() - start_time)))
+ return X_train, y_train, X_test, y_test
+
+
+def load_ptb_dataset(path='data'):
+ """Load Penn TreeBank (PTB) dataset.
+
+ It is used in many LANGUAGE MODELING papers,
+ including "Empirical Evaluation and Combination of Advanced Language
+ Modeling Techniques", "Recurrent Neural Network Regularization".
+ It consists of 929k training words, 73k validation words, and 82k test
+ words. It has 10k words in its vocabulary.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/ptb/``.
+
+ Returns
+ --------
+ train_data, valid_data, test_data : list of int
+ The training, validating and testing data in integer format.
+ vocab_size : int
+ The vocabulary size.
+
+ Examples
+ --------
+ >>> train_data, valid_data, test_data, vocab_size = tl.files.load_ptb_dataset()
+
+ References
+ ---------------
+ - ``tensorflow.models.rnn.ptb import reader``
+ - `Manual download `__
+
+ Notes
+ ------
+ - If you want to get the raw data, see the source code.
+
+ """
+ path = os.path.join(path, 'ptb')
+ logging.info("Load or Download Penn TreeBank (PTB) dataset > {}".format(path))
+
+ # Maybe dowload and uncompress tar, or load exsisting files
+ filename = 'simple-examples.tgz'
+ url = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/'
+ maybe_download_and_extract(filename, path, url, extract=True)
+
+ data_path = os.path.join(path, 'simple-examples', 'data')
+ train_path = os.path.join(data_path, "ptb.train.txt")
+ valid_path = os.path.join(data_path, "ptb.valid.txt")
+ test_path = os.path.join(data_path, "ptb.test.txt")
+
+ word_to_id = nlp.build_vocab(nlp.read_words(train_path))
+
+ train_data = nlp.words_to_word_ids(nlp.read_words(train_path), word_to_id)
+ valid_data = nlp.words_to_word_ids(nlp.read_words(valid_path), word_to_id)
+ test_data = nlp.words_to_word_ids(nlp.read_words(test_path), word_to_id)
+ vocab_size = len(word_to_id)
+
+ # logging.info(nlp.read_words(train_path)) # ... 'according', 'to', 'mr.', '', '']
+ # logging.info(train_data) # ... 214, 5, 23, 1, 2]
+ # logging.info(word_to_id) # ... 'beyond': 1295, 'anti-nuclear': 9599, 'trouble': 1520, '': 2 ... }
+ # logging.info(vocabulary) # 10000
+ # exit()
+ return train_data, valid_data, test_data, vocab_size
+
+
+def load_matt_mahoney_text8_dataset(path='data'):
+ """Load Matt Mahoney's dataset.
+
+ Download a text file from Matt Mahoney's website
+ if not present, and make sure it's the right size.
+ Extract the first file enclosed in a zip file as a list of words.
+ This dataset can be used for Word Embedding.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/mm_test8/``.
+
+ Returns
+ --------
+ list of str
+ The raw text data e.g. [.... 'their', 'families', 'who', 'were', 'expelled', 'from', 'jerusalem', ...]
+
+ Examples
+ --------
+ >>> words = tl.files.load_matt_mahoney_text8_dataset()
+ >>> print('Data size', len(words))
+
+ """
+ path = os.path.join(path, 'mm_test8')
+ logging.info("Load or Download matt_mahoney_text8 Dataset> {}".format(path))
+
+ filename = 'text8.zip'
+ url = 'http://mattmahoney.net/dc/'
+ maybe_download_and_extract(filename, path, url, expected_bytes=31344016)
+
+ with zipfile.ZipFile(os.path.join(path, filename)) as f:
+ word_list = f.read(f.namelist()[0]).split()
+ for idx, _ in enumerate(word_list):
+ word_list[idx] = word_list[idx].decode()
+ return word_list
+
+
+def load_imdb_dataset(
+ path='data', nb_words=None, skip_top=0, maxlen=None, test_split=0.2, seed=113, start_char=1, oov_char=2,
+ index_from=3
+):
+ """Load IMDB dataset.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/imdb/``.
+ nb_words : int
+ Number of words to get.
+ skip_top : int
+ Top most frequent words to ignore (they will appear as oov_char value in the sequence data).
+ maxlen : int
+ Maximum sequence length. Any longer sequence will be truncated.
+ seed : int
+ Seed for reproducible data shuffling.
+ start_char : int
+ The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character.
+ oov_char : int
+ Words that were cut out because of the num_words or skip_top limit will be replaced with this character.
+ index_from : int
+ Index actual words with this index and higher.
+
+ Examples
+ --------
+ >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(
+ ... nb_words=20000, test_split=0.2)
+ >>> print('X_train.shape', X_train.shape)
+ (20000,) [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
+ >>> print('y_train.shape', y_train.shape)
+ (20000,) [1 0 0 ..., 1 0 1]
+
+ References
+ -----------
+ - `Modified from keras. `__
+
+ """
+ path = os.path.join(path, 'imdb')
+
+ filename = "imdb.pkl"
+ url = 'https://s3.amazonaws.com/text-datasets/'
+ maybe_download_and_extract(filename, path, url)
+
+ if filename.endswith(".gz"):
+ f = gzip.open(os.path.join(path, filename), 'rb')
+ else:
+ f = open(os.path.join(path, filename), 'rb')
+
+ X, labels = cPickle.load(f)
+ f.close()
+
+ np.random.seed(seed)
+ np.random.shuffle(X)
+ np.random.seed(seed)
+ np.random.shuffle(labels)
+
+ if start_char is not None:
+ X = [[start_char] + [w + index_from for w in x] for x in X]
+ elif index_from:
+ X = [[w + index_from for w in x] for x in X]
+
+ if maxlen:
+ new_X = []
+ new_labels = []
+ for x, y in zip(X, labels):
+ if len(x) < maxlen:
+ new_X.append(x)
+ new_labels.append(y)
+ X = new_X
+ labels = new_labels
+ if not X:
+ raise Exception(
+ 'After filtering for sequences shorter than maxlen=' + str(maxlen) + ', no sequence was kept. '
+ 'Increase maxlen.'
+ )
+ if not nb_words:
+ nb_words = max([max(x) for x in X])
+
+ # by convention, use 2 as OOV word
+ # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
+ if oov_char is not None:
+ X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
+ else:
+ nX = []
+ for x in X:
+ nx = []
+ for w in x:
+ if (w >= nb_words or w < skip_top):
+ nx.append(w)
+ nX.append(nx)
+ X = nX
+
+ X_train = np.array(X[:int(len(X) * (1 - test_split))])
+ y_train = np.array(labels[:int(len(X) * (1 - test_split))])
+
+ X_test = np.array(X[int(len(X) * (1 - test_split)):])
+ y_test = np.array(labels[int(len(X) * (1 - test_split)):])
+
+ return X_train, y_train, X_test, y_test
+
+
+def load_nietzsche_dataset(path='data'):
+ """Load Nietzsche dataset.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/nietzsche/``.
+
+ Returns
+ --------
+ str
+ The content.
+
+ Examples
+ --------
+ >>> see tutorial_generate_text.py
+ >>> words = tl.files.load_nietzsche_dataset()
+ >>> words = basic_clean_str(words)
+ >>> words = words.split()
+
+ """
+ logging.info("Load or Download nietzsche dataset > {}".format(path))
+ path = os.path.join(path, 'nietzsche')
+
+ filename = "nietzsche.txt"
+ url = 'https://s3.amazonaws.com/text-datasets/'
+ filepath = maybe_download_and_extract(filename, path, url)
+
+ with open(filepath, "r") as f:
+ words = f.read()
+ return words
+
+
+def load_wmt_en_fr_dataset(path='data'):
+ """Load WMT'15 English-to-French translation dataset.
+
+ It will download the data from the WMT'15 Website (10^9-French-English corpus), and the 2013 news test from the same site as development set.
+ Returns the directories of training data and test data.
+
+ Parameters
+ ----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/wmt_en_fr/``.
+
+ References
+ ----------
+ - Code modified from /tensorflow/models/rnn/translation/data_utils.py
+
+ Notes
+ -----
+ Usually, it will take a long time to download this dataset.
+
+ """
+ path = os.path.join(path, 'wmt_en_fr')
+ # URLs for WMT data.
+ _WMT_ENFR_TRAIN_URL = "http://www.statmt.org/wmt10/"
+ _WMT_ENFR_DEV_URL = "http://www.statmt.org/wmt15/"
+
+ def gunzip_file(gz_path, new_path):
+ """Unzips from gz_path into new_path."""
+ logging.info("Unpacking %s to %s" % (gz_path, new_path))
+ with gzip.open(gz_path, "rb") as gz_file:
+ with open(new_path, "wb") as new_file:
+ for line in gz_file:
+ new_file.write(line)
+
+ def get_wmt_enfr_train_set(path):
+ """Download the WMT en-fr training corpus to directory unless it's there."""
+ filename = "training-giga-fren.tar"
+ maybe_download_and_extract(filename, path, _WMT_ENFR_TRAIN_URL, extract=True)
+ train_path = os.path.join(path, "giga-fren.release2.fixed")
+ gunzip_file(train_path + ".fr.gz", train_path + ".fr")
+ gunzip_file(train_path + ".en.gz", train_path + ".en")
+ return train_path
+
+ def get_wmt_enfr_dev_set(path):
+ """Download the WMT en-fr training corpus to directory unless it's there."""
+ filename = "dev-v2.tgz"
+ dev_file = maybe_download_and_extract(filename, path, _WMT_ENFR_DEV_URL, extract=False)
+ dev_name = "newstest2013"
+ dev_path = os.path.join(path, "newstest2013")
+ if not (gfile.Exists(dev_path + ".fr") and gfile.Exists(dev_path + ".en")):
+ logging.info("Extracting tgz file %s" % dev_file)
+ with tarfile.open(dev_file, "r:gz") as dev_tar:
+ fr_dev_file = dev_tar.getmember("dev/" + dev_name + ".fr")
+ en_dev_file = dev_tar.getmember("dev/" + dev_name + ".en")
+ fr_dev_file.name = dev_name + ".fr" # Extract without "dev/" prefix.
+ en_dev_file.name = dev_name + ".en"
+ dev_tar.extract(fr_dev_file, path)
+ dev_tar.extract(en_dev_file, path)
+ return dev_path
+
+ logging.info("Load or Download WMT English-to-French translation > {}".format(path))
+
+ train_path = get_wmt_enfr_train_set(path)
+ dev_path = get_wmt_enfr_dev_set(path)
+
+ return train_path, dev_path
+
+
+def load_flickr25k_dataset(tag='sky', path="data", n_threads=50, printable=False):
+ """Load Flickr25K dataset.
+
+ Returns a list of images by a given tag from Flick25k dataset,
+ it will download Flickr25k from `the official website `__
+ at the first time you use it.
+
+ Parameters
+ ------------
+ tag : str or None
+ What images to return.
+ - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search `__.
+ - If you want to get all images, set to ``None``.
+
+ path : str
+ The path that the data is downloaded to, defaults is ``data/flickr25k/``.
+ n_threads : int
+ The number of thread to read image.
+ printable : boolean
+ Whether to print infomation when reading images, default is ``False``.
+
+ Examples
+ -----------
+ Get images with tag of sky
+
+ >>> images = tl.files.load_flickr25k_dataset(tag='sky')
+
+ Get all images
+
+ >>> images = tl.files.load_flickr25k_dataset(tag=None, n_threads=100, printable=True)
+
+ """
+ path = os.path.join(path, 'flickr25k')
+
+ filename = 'mirflickr25k.zip'
+ url = 'http://press.liacs.nl/mirflickr/mirflickr25k/'
+
+ # download dataset
+ if folder_exists(os.path.join(path, "mirflickr")) is False:
+ logging.info("[*] Flickr25k is nonexistent in {}".format(path))
+ maybe_download_and_extract(filename, path, url, extract=True)
+ del_file(os.path.join(path, filename))
+
+ # return images by the given tag.
+ # 1. image path list
+ folder_imgs = os.path.join(path, "mirflickr")
+ path_imgs = load_file_list(path=folder_imgs, regx='\\.jpg', printable=False)
+ path_imgs.sort(key=natural_keys)
+
+ # 2. tag path list
+ folder_tags = os.path.join(path, "mirflickr", "meta", "tags")
+ path_tags = load_file_list(path=folder_tags, regx='\\.txt', printable=False)
+ path_tags.sort(key=natural_keys)
+
+ # 3. select images
+ if tag is None:
+ logging.info("[Flickr25k] reading all images")
+ else:
+ logging.info("[Flickr25k] reading images with tag: {}".format(tag))
+ images_list = []
+ for idx, _v in enumerate(path_tags):
+ tags = read_file(os.path.join(folder_tags, path_tags[idx])).split('\n')
+ # logging.info(idx+1, tags)
+ if tag is None or tag in tags:
+ images_list.append(path_imgs[idx])
+
+ images = visualize.read_images(images_list, folder_imgs, n_threads=n_threads, printable=printable)
+ return images
+
+
+def load_flickr1M_dataset(tag='sky', size=10, path="data", n_threads=50, printable=False):
+ """Load Flick1M dataset.
+
+ Returns a list of images by a given tag from Flickr1M dataset,
+ it will download Flickr1M from `the official website `__
+ at the first time you use it.
+
+ Parameters
+ ------------
+ tag : str or None
+ What images to return.
+ - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search `__.
+ - If you want to get all images, set to ``None``.
+
+ size : int
+ integer between 1 to 10. 1 means 100k images ... 5 means 500k images, 10 means all 1 million images. Default is 10.
+ path : str
+ The path that the data is downloaded to, defaults is ``data/flickr25k/``.
+ n_threads : int
+ The number of thread to read image.
+ printable : boolean
+ Whether to print infomation when reading images, default is ``False``.
+
+ Examples
+ ----------
+ Use 200k images
+
+ >>> images = tl.files.load_flickr1M_dataset(tag='zebra', size=2)
+
+ Use 1 Million images
+
+ >>> images = tl.files.load_flickr1M_dataset(tag='zebra')
+
+ """
+ path = os.path.join(path, 'flickr1M')
+ logging.info("[Flickr1M] using {}% of images = {}".format(size * 10, size * 100000))
+ images_zip = [
+ 'images0.zip', 'images1.zip', 'images2.zip', 'images3.zip', 'images4.zip', 'images5.zip', 'images6.zip',
+ 'images7.zip', 'images8.zip', 'images9.zip'
+ ]
+ tag_zip = 'tags.zip'
+ url = 'http://press.liacs.nl/mirflickr/mirflickr1m/'
+
+ # download dataset
+ for image_zip in images_zip[0:size]:
+ image_folder = image_zip.split(".")[0]
+ # logging.info(path+"/"+image_folder)
+ if folder_exists(os.path.join(path, image_folder)) is False:
+ # logging.info(image_zip)
+ logging.info("[Flickr1M] {} is missing in {}".format(image_folder, path))
+ maybe_download_and_extract(image_zip, path, url, extract=True)
+ del_file(os.path.join(path, image_zip))
+ # os.system("mv {} {}".format(os.path.join(path, 'images'), os.path.join(path, image_folder)))
+ shutil.move(os.path.join(path, 'images'), os.path.join(path, image_folder))
+ else:
+ logging.info("[Flickr1M] {} exists in {}".format(image_folder, path))
+
+ # download tag
+ if folder_exists(os.path.join(path, "tags")) is False:
+ logging.info("[Flickr1M] tag files is nonexistent in {}".format(path))
+ maybe_download_and_extract(tag_zip, path, url, extract=True)
+ del_file(os.path.join(path, tag_zip))
+ else:
+ logging.info("[Flickr1M] tags exists in {}".format(path))
+
+ # 1. image path list
+ images_list = []
+ images_folder_list = []
+ for i in range(0, size):
+ images_folder_list += load_folder_list(path=os.path.join(path, 'images%d' % i))
+ images_folder_list.sort(key=lambda s: int(s.split('/')[-1])) # folder/images/ddd
+
+ for folder in images_folder_list[0:size * 10]:
+ tmp = load_file_list(path=folder, regx='\\.jpg', printable=False)
+ tmp.sort(key=lambda s: int(s.split('.')[-2])) # ddd.jpg
+ images_list.extend([os.path.join(folder, x) for x in tmp])
+
+ # 2. tag path list
+ tag_list = []
+ tag_folder_list = load_folder_list(os.path.join(path, "tags"))
+
+ # tag_folder_list.sort(key=lambda s: int(s.split("/")[-1])) # folder/images/ddd
+ tag_folder_list.sort(key=lambda s: int(os.path.basename(s)))
+
+ for folder in tag_folder_list[0:size * 10]:
+ tmp = load_file_list(path=folder, regx='\\.txt', printable=False)
+ tmp.sort(key=lambda s: int(s.split('.')[-2])) # ddd.txt
+ tmp = [os.path.join(folder, s) for s in tmp]
+ tag_list += tmp
+
+ # 3. select images
+ logging.info("[Flickr1M] searching tag: {}".format(tag))
+ select_images_list = []
+ for idx, _val in enumerate(tag_list):
+ tags = read_file(tag_list[idx]).split('\n')
+ if tag in tags:
+ select_images_list.append(images_list[idx])
+
+ logging.info("[Flickr1M] reading images with tag: {}".format(tag))
+ images = visualize.read_images(select_images_list, '', n_threads=n_threads, printable=printable)
+ return images
+
+
+def load_cyclegan_dataset(filename='summer2winter_yosemite', path='data'):
+ """Load images from CycleGAN's database, see `this link `__.
+
+ Parameters
+ ------------
+ filename : str
+ The dataset you want, see `this link `__.
+ path : str
+ The path that the data is downloaded to, defaults is `data/cyclegan`
+
+ Examples
+ ---------
+ >>> im_train_A, im_train_B, im_test_A, im_test_B = load_cyclegan_dataset(filename='summer2winter_yosemite')
+
+ """
+ path = os.path.join(path, 'cyclegan')
+ url = 'https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/'
+
+ if folder_exists(os.path.join(path, filename)) is False:
+ logging.info("[*] {} is nonexistent in {}".format(filename, path))
+ maybe_download_and_extract(filename + '.zip', path, url, extract=True)
+ del_file(os.path.join(path, filename + '.zip'))
+
+ def load_image_from_folder(path):
+ path_imgs = load_file_list(path=path, regx='\\.jpg', printable=False)
+ return visualize.read_images(path_imgs, path=path, n_threads=10, printable=False)
+
+ im_train_A = load_image_from_folder(os.path.join(path, filename, "trainA"))
+ im_train_B = load_image_from_folder(os.path.join(path, filename, "trainB"))
+ im_test_A = load_image_from_folder(os.path.join(path, filename, "testA"))
+ im_test_B = load_image_from_folder(os.path.join(path, filename, "testB"))
+
+ def if_2d_to_3d(images): # [h, w] --> [h, w, 3]
+ for i, _v in enumerate(images):
+ if len(images[i].shape) == 2:
+ images[i] = images[i][:, :, np.newaxis]
+ images[i] = np.tile(images[i], (1, 1, 3))
+ return images
+
+ im_train_A = if_2d_to_3d(im_train_A)
+ im_train_B = if_2d_to_3d(im_train_B)
+ im_test_A = if_2d_to_3d(im_test_A)
+ im_test_B = if_2d_to_3d(im_test_B)
+
+ return im_train_A, im_train_B, im_test_A, im_test_B
+
+
+def download_file_from_google_drive(ID, destination):
+ """Download file from Google Drive.
+
+ See ``tl.files.load_celebA_dataset`` for example.
+
+ Parameters
+ --------------
+ ID : str
+ The driver ID.
+ destination : str
+ The destination for save file.
+
+ """
+ try:
+ from tqdm import tqdm
+ except ImportError as e:
+ print(e)
+ raise ImportError("Module tqdm not found. Please install tqdm via pip or other package managers.")
+
+ try:
+ import requests
+ except ImportError as e:
+ print(e)
+ raise ImportError("Module requests not found. Please install requests via pip or other package managers.")
+
+ def save_response_content(response, destination, chunk_size=32 * 1024):
+
+ total_size = int(response.headers.get('content-length', 0))
+ with open(destination, "wb") as f:
+ for chunk in tqdm(response.iter_content(chunk_size), total=total_size, unit='B', unit_scale=True,
+ desc=destination):
+ if chunk: # filter out keep-alive new chunks
+ f.write(chunk)
+
+ def get_confirm_token(response):
+ for key, value in response.cookies.items():
+ if key.startswith('download_warning'):
+ return value
+ return None
+
+ URL = "https://docs.google.com/uc?export=download"
+ session = requests.Session()
+
+ response = session.get(URL, params={'id': ID}, stream=True)
+ token = get_confirm_token(response)
+
+ if token:
+ params = {'id': ID, 'confirm': token}
+ response = session.get(URL, params=params, stream=True)
+ save_response_content(response, destination)
+
+
+def load_celebA_dataset(path='data'):
+ """Load CelebA dataset
+
+ Return a list of image path.
+
+ Parameters
+ -----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/celebA/``.
+
+ """
+ data_dir = 'celebA'
+ filename, drive_id = "img_align_celeba.zip", "0B7EVK8r0v71pZjFTYXZWM3FlRnM"
+ save_path = os.path.join(path, filename)
+ image_path = os.path.join(path, data_dir)
+ if os.path.exists(image_path):
+ logging.info('[*] {} already exists'.format(save_path))
+ else:
+ exists_or_mkdir(path)
+ download_file_from_google_drive(drive_id, save_path)
+ zip_dir = ''
+ with zipfile.ZipFile(save_path) as zf:
+ zip_dir = zf.namelist()[0]
+ zf.extractall(path)
+ os.remove(save_path)
+ os.rename(os.path.join(path, zip_dir), image_path)
+
+ data_files = load_file_list(path=image_path, regx='\\.jpg', printable=False)
+ for i, _v in enumerate(data_files):
+ data_files[i] = os.path.join(image_path, data_files[i])
+ return data_files
+
+
+def load_voc_dataset(path='data', dataset='2012', contain_classes_in_person=False):
+ """Pascal VOC 2007/2012 Dataset.
+
+ It has 20 objects:
+ aeroplane, bicycle, bird, boat, bottle, bus, car, cat, chair, cow, diningtable, dog, horse, motorbike, person, pottedplant, sheep, sofa, train, tvmonitor
+ and additional 3 classes : head, hand, foot for person.
+
+ Parameters
+ -----------
+ path : str
+ The path that the data is downloaded to, defaults is ``data/VOC``.
+ dataset : str
+ The VOC dataset version, `2012`, `2007`, `2007test` or `2012test`. We usually train model on `2007+2012` and test it on `2007test`.
+ contain_classes_in_person : boolean
+ Whether include head, hand and foot annotation, default is False.
+
+ Returns
+ ---------
+ imgs_file_list : list of str
+ Full paths of all images.
+ imgs_semseg_file_list : list of str
+ Full paths of all maps for semantic segmentation. Note that not all images have this map!
+ imgs_insseg_file_list : list of str
+ Full paths of all maps for instance segmentation. Note that not all images have this map!
+ imgs_ann_file_list : list of str
+ Full paths of all annotations for bounding box and object class, all images have this annotations.
+ classes : list of str
+ Classes in order.
+ classes_in_person : list of str
+ Classes in person.
+ classes_dict : dictionary
+ Class label to integer.
+ n_objs_list : list of int
+ Number of objects in all images in ``imgs_file_list`` in order.
+ objs_info_list : list of str
+ Darknet format for the annotation of all images in ``imgs_file_list`` in order. ``[class_id x_centre y_centre width height]`` in ratio format.
+ objs_info_dicts : dictionary
+ The annotation of all images in ``imgs_file_list``, ``{imgs_file_list : dictionary for annotation}``,
+ format from `TensorFlow/Models/object-detection `__.
+
+ Examples
+ ----------
+ >>> imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list,
+ >>> classes, classes_in_person, classes_dict,
+ >>> n_objs_list, objs_info_list, objs_info_dicts = tl.files.load_voc_dataset(dataset="2012", contain_classes_in_person=False)
+ >>> idx = 26
+ >>> print(classes)
+ ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
+ >>> print(classes_dict)
+ {'sheep': 16, 'horse': 12, 'bicycle': 1, 'bottle': 4, 'cow': 9, 'sofa': 17, 'car': 6, 'dog': 11, 'cat': 7, 'person': 14, 'train': 18, 'diningtable': 10, 'aeroplane': 0, 'bus': 5, 'pottedplant': 15, 'tvmonitor': 19, 'chair': 8, 'bird': 2, 'boat': 3, 'motorbike': 13}
+ >>> print(imgs_file_list[idx])
+ data/VOC/VOC2012/JPEGImages/2007_000423.jpg
+ >>> print(n_objs_list[idx])
+ 2
+ >>> print(imgs_ann_file_list[idx])
+ data/VOC/VOC2012/Annotations/2007_000423.xml
+ >>> print(objs_info_list[idx])
+ 14 0.173 0.461333333333 0.142 0.496
+ 14 0.828 0.542666666667 0.188 0.594666666667
+ >>> ann = tl.prepro.parse_darknet_ann_str_to_list(objs_info_list[idx])
+ >>> print(ann)
+ [[14, 0.173, 0.461333333333, 0.142, 0.496], [14, 0.828, 0.542666666667, 0.188, 0.594666666667]]
+ >>> c, b = tl.prepro.parse_darknet_ann_list_to_cls_box(ann)
+ >>> print(c, b)
+ [14, 14] [[0.173, 0.461333333333, 0.142, 0.496], [0.828, 0.542666666667, 0.188, 0.594666666667]]
+
+ References
+ -------------
+ - `Pascal VOC2012 Website `__.
+ - `Pascal VOC2007 Website `__.
+
+ """
+
+ import xml.etree.ElementTree as ET
+
+ try:
+ import lxml.etree as etree
+ except ImportError as e:
+ print(e)
+ raise ImportError("Module lxml not found. Please install lxml via pip or other package managers.")
+
+ path = os.path.join(path, 'VOC')
+
+ def _recursive_parse_xml_to_dict(xml):
+ """Recursively parses XML contents to python dict.
+
+ We assume that `object` tags are the only ones that can appear
+ multiple times at the same level of a tree.
+
+ Args:
+ xml: xml tree obtained by parsing XML file contents using lxml.etree
+
+ Returns:
+ Python dictionary holding XML contents.
+
+ """
+ if not xml:
+ # if xml is not None:
+ return {xml.tag: xml.text}
+ result = {}
+ for child in xml:
+ child_result = _recursive_parse_xml_to_dict(child)
+ if child.tag != 'object':
+ result[child.tag] = child_result[child.tag]
+ else:
+ if child.tag not in result:
+ result[child.tag] = []
+ result[child.tag].append(child_result[child.tag])
+ return {xml.tag: result}
+
+ if dataset == "2012":
+ url = "http://host.robots.ox.ac.uk/pascal/VOC/voc2012/"
+ tar_filename = "VOCtrainval_11-May-2012.tar"
+ extracted_filename = "VOC2012" # "VOCdevkit/VOC2012"
+ logging.info(" [============= VOC 2012 =============]")
+ elif dataset == "2012test":
+ extracted_filename = "VOC2012test" # "VOCdevkit/VOC2012"
+ logging.info(" [============= VOC 2012 Test Set =============]")
+ logging.info(
+ " \nAuthor: 2012test only have person annotation, so 2007test is highly recommended for testing !\n"
+ )
+ time.sleep(3)
+ if os.path.isdir(os.path.join(path, extracted_filename)) is False:
+ logging.info("For VOC 2012 Test data - online registration required")
+ logging.info(
+ " Please download VOC2012test.tar from: \n register: http://host.robots.ox.ac.uk:8080 \n voc2012 : http://host.robots.ox.ac.uk:8080/eval/challenges/voc2012/ \ndownload: http://host.robots.ox.ac.uk:8080/eval/downloads/VOC2012test.tar"
+ )
+ logging.info(" unzip VOC2012test.tar,rename the folder to VOC2012test and put it into %s" % path)
+ exit()
+ # # http://host.robots.ox.ac.uk:8080/eval/downloads/VOC2012test.tar
+ # url = "http://host.robots.ox.ac.uk:8080/eval/downloads/"
+ # tar_filename = "VOC2012test.tar"
+ elif dataset == "2007":
+ url = "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/"
+ tar_filename = "VOCtrainval_06-Nov-2007.tar"
+ extracted_filename = "VOC2007"
+ logging.info(" [============= VOC 2007 =============]")
+ elif dataset == "2007test":
+ # http://host.robots.ox.ac.uk/pascal/VOC/voc2007/index.html#testdata
+ # http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar
+ url = "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/"
+ tar_filename = "VOCtest_06-Nov-2007.tar"
+ extracted_filename = "VOC2007test"
+ logging.info(" [============= VOC 2007 Test Set =============]")
+ else:
+ raise Exception("Please set the dataset aug to 2012, 2012test or 2007.")
+
+ # download dataset
+ if dataset != "2012test":
+ _platform = sys.platform
+ if folder_exists(os.path.join(path, extracted_filename)) is False:
+ logging.info("[VOC] {} is nonexistent in {}".format(extracted_filename, path))
+ maybe_download_and_extract(tar_filename, path, url, extract=True)
+ del_file(os.path.join(path, tar_filename))
+ if dataset == "2012":
+ if _platform == "win32":
+ os.system("mv {}\VOCdevkit\VOC2012 {}\VOC2012".format(path, path))
+ else:
+ os.system("mv {}/VOCdevkit/VOC2012 {}/VOC2012".format(path, path))
+ elif dataset == "2007":
+ if _platform == "win32":
+ os.system("mv {}\VOCdevkit\VOC2007 {}\VOC2007".format(path, path))
+ else:
+ os.system("mv {}/VOCdevkit/VOC2007 {}/VOC2007".format(path, path))
+ elif dataset == "2007test":
+ if _platform == "win32":
+ os.system("mv {}\VOCdevkit\VOC2007 {}\VOC2007test".format(path, path))
+ else:
+ os.system("mv {}/VOCdevkit/VOC2007 {}/VOC2007test".format(path, path))
+ del_folder(os.path.join(path, 'VOCdevkit'))
+ # object classes(labels) NOTE: YOU CAN CUSTOMIZE THIS LIST
+ classes = [
+ "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog",
+ "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"
+ ]
+ if contain_classes_in_person:
+ classes_in_person = ["head", "hand", "foot"]
+ else:
+ classes_in_person = []
+
+ classes += classes_in_person # use extra 3 classes for person
+
+ classes_dict = utils.list_string_to_dict(classes)
+ logging.info("[VOC] object classes {}".format(classes_dict))
+
+ # 1. image path list
+ # folder_imgs = path+"/"+extracted_filename+"/JPEGImages/"
+ folder_imgs = os.path.join(path, extracted_filename, "JPEGImages")
+ imgs_file_list = load_file_list(path=folder_imgs, regx='\\.jpg', printable=False)
+ logging.info("[VOC] {} images found".format(len(imgs_file_list)))
+
+ imgs_file_list.sort(
+ key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])
+ ) # 2007_000027.jpg --> 2007000027
+
+ imgs_file_list = [os.path.join(folder_imgs, s) for s in imgs_file_list]
+ # logging.info('IM',imgs_file_list[0::3333], imgs_file_list[-1])
+ if dataset != "2012test":
+ # ======== 2. semantic segmentation maps path list
+ # folder_semseg = path+"/"+extracted_filename+"/SegmentationClass/"
+ folder_semseg = os.path.join(path, extracted_filename, "SegmentationClass")
+ imgs_semseg_file_list = load_file_list(path=folder_semseg, regx='\\.png', printable=False)
+ logging.info("[VOC] {} maps for semantic segmentation found".format(len(imgs_semseg_file_list)))
+ imgs_semseg_file_list.sort(
+ key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])
+ ) # 2007_000032.png --> 2007000032
+ imgs_semseg_file_list = [os.path.join(folder_semseg, s) for s in imgs_semseg_file_list]
+ # logging.info('Semantic Seg IM',imgs_semseg_file_list[0::333], imgs_semseg_file_list[-1])
+ # ======== 3. instance segmentation maps path list
+ # folder_insseg = path+"/"+extracted_filename+"/SegmentationObject/"
+ folder_insseg = os.path.join(path, extracted_filename, "SegmentationObject")
+ imgs_insseg_file_list = load_file_list(path=folder_insseg, regx='\\.png', printable=False)
+ logging.info("[VOC] {} maps for instance segmentation found".format(len(imgs_semseg_file_list)))
+ imgs_insseg_file_list.sort(
+ key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])
+ ) # 2007_000032.png --> 2007000032
+ imgs_insseg_file_list = [os.path.join(folder_insseg, s) for s in imgs_insseg_file_list]
+ # logging.info('Instance Seg IM',imgs_insseg_file_list[0::333], imgs_insseg_file_list[-1])
+ else:
+ imgs_semseg_file_list = []
+ imgs_insseg_file_list = []
+ # 4. annotations for bounding box and object class
+ # folder_ann = path+"/"+extracted_filename+"/Annotations/"
+ folder_ann = os.path.join(path, extracted_filename, "Annotations")
+ imgs_ann_file_list = load_file_list(path=folder_ann, regx='\\.xml', printable=False)
+ logging.info(
+ "[VOC] {} XML annotation files for bounding box and object class found".format(len(imgs_ann_file_list))
+ )
+ imgs_ann_file_list.sort(
+ key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])
+ ) # 2007_000027.xml --> 2007000027
+ imgs_ann_file_list = [os.path.join(folder_ann, s) for s in imgs_ann_file_list]
+ # logging.info('ANN',imgs_ann_file_list[0::3333], imgs_ann_file_list[-1])
+
+ if dataset == "2012test": # remove unused images in JPEG folder
+ imgs_file_list_new = []
+ for ann in imgs_ann_file_list:
+ ann = os.path.split(ann)[-1].split('.')[0]
+ for im in imgs_file_list:
+ if ann in im:
+ imgs_file_list_new.append(im)
+ break
+ imgs_file_list = imgs_file_list_new
+ logging.info("[VOC] keep %d images" % len(imgs_file_list_new))
+
+ # parse XML annotations
+ def convert(size, box):
+ dw = 1. / size[0]
+ dh = 1. / size[1]
+ x = (box[0] + box[1]) / 2.0
+ y = (box[2] + box[3]) / 2.0
+ w = box[1] - box[0]
+ h = box[3] - box[2]
+ x = x * dw
+ w = w * dw
+ y = y * dh
+ h = h * dh
+ return x, y, w, h
+
+ def convert_annotation(file_name):
+ """Given VOC2012 XML Annotations, returns number of objects and info."""
+ in_file = open(file_name)
+ out_file = ""
+ tree = ET.parse(in_file)
+ root = tree.getroot()
+ size = root.find('size')
+ w = int(size.find('width').text)
+ h = int(size.find('height').text)
+ n_objs = 0
+
+ for obj in root.iter('object'):
+ if dataset != "2012test":
+ difficult = obj.find('difficult').text
+ cls = obj.find('name').text
+ if cls not in classes or int(difficult) == 1:
+ continue
+ else:
+ cls = obj.find('name').text
+ if cls not in classes:
+ continue
+ cls_id = classes.index(cls)
+ xmlbox = obj.find('bndbox')
+ b = (
+ float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),
+ float(xmlbox.find('ymax').text)
+ )
+ bb = convert((w, h), b)
+
+ out_file += str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n'
+ n_objs += 1
+ if cls in "person":
+ for part in obj.iter('part'):
+ cls = part.find('name').text
+ if cls not in classes_in_person:
+ continue
+ cls_id = classes.index(cls)
+ xmlbox = part.find('bndbox')
+ b = (
+ float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text),
+ float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text)
+ )
+ bb = convert((w, h), b)
+ # out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
+ out_file += str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n'
+ n_objs += 1
+ in_file.close()
+ return n_objs, out_file
+
+ logging.info("[VOC] Parsing xml annotations files")
+ n_objs_list = []
+ objs_info_list = [] # Darknet Format list of string
+ objs_info_dicts = {}
+ for idx, ann_file in enumerate(imgs_ann_file_list):
+ n_objs, objs_info = convert_annotation(ann_file)
+ n_objs_list.append(n_objs)
+ objs_info_list.append(objs_info)
+ with tf.io.gfile.GFile(ann_file, 'r') as fid:
+ xml_str = fid.read()
+ xml = etree.fromstring(xml_str)
+ data = _recursive_parse_xml_to_dict(xml)['annotation']
+ objs_info_dicts.update({imgs_file_list[idx]: data})
+
+ return imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list, classes, classes_in_person, classes_dict, n_objs_list, objs_info_list, objs_info_dicts
+
+
+def load_mpii_pose_dataset(path='data', is_16_pos_only=False):
+ """Load MPII Human Pose Dataset.
+
+ Parameters
+ -----------
+ path : str
+ The path that the data is downloaded to.
+ is_16_pos_only : boolean
+ If True, only return the peoples contain 16 pose keypoints. (Usually be used for single person pose estimation)
+
+ Returns
+ ----------
+ img_train_list : list of str
+ The image directories of training data.
+ ann_train_list : list of dict
+ The annotations of training data.
+ img_test_list : list of str
+ The image directories of testing data.
+ ann_test_list : list of dict
+ The annotations of testing data.
+
+ Examples
+ --------
+ >>> import pprint
+ >>> import tensorlayer as tl
+ >>> img_train_list, ann_train_list, img_test_list, ann_test_list = tl.files.load_mpii_pose_dataset()
+ >>> image = tl.vis.read_image(img_train_list[0])
+ >>> tl.vis.draw_mpii_pose_to_image(image, ann_train_list[0], 'image.png')
+ >>> pprint.pprint(ann_train_list[0])
+
+ References
+ -----------
+ - `MPII Human Pose Dataset. CVPR 14 `__
+ - `MPII Human Pose Models. CVPR 16 `__
+ - `MPII Human Shape, Poselet Conditioned Pictorial Structures and etc `__
+ - `MPII Keyponts and ID `__
+ """
+ path = os.path.join(path, 'mpii_human_pose')
+ logging.info("Load or Download MPII Human Pose > {}".format(path))
+
+ # annotation
+ url = "http://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/"
+ tar_filename = "mpii_human_pose_v1_u12_2.zip"
+ extracted_filename = "mpii_human_pose_v1_u12_2"
+ if folder_exists(os.path.join(path, extracted_filename)) is False:
+ logging.info("[MPII] (annotation) {} is nonexistent in {}".format(extracted_filename, path))
+ maybe_download_and_extract(tar_filename, path, url, extract=True)
+ del_file(os.path.join(path, tar_filename))
+
+ # images
+ url = "http://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/"
+ tar_filename = "mpii_human_pose_v1.tar.gz"
+ extracted_filename2 = "images"
+ if folder_exists(os.path.join(path, extracted_filename2)) is False:
+ logging.info("[MPII] (images) {} is nonexistent in {}".format(extracted_filename, path))
+ maybe_download_and_extract(tar_filename, path, url, extract=True)
+ del_file(os.path.join(path, tar_filename))
+
+ # parse annotation, format see http://human-pose.mpi-inf.mpg.de/#download
+ logging.info("reading annotations from mat file ...")
+ # mat = sio.loadmat(os.path.join(path, extracted_filename, "mpii_human_pose_v1_u12_1.mat"))
+
+ # def fix_wrong_joints(joint): # https://github.com/mitmul/deeppose/blob/master/datasets/mpii_dataset.py
+ # if '12' in joint and '13' in joint and '2' in joint and '3' in joint:
+ # if ((joint['12'][0] < joint['13'][0]) and
+ # (joint['3'][0] < joint['2'][0])):
+ # joint['2'], joint['3'] = joint['3'], joint['2']
+ # if ((joint['12'][0] > joint['13'][0]) and
+ # (joint['3'][0] > joint['2'][0])):
+ # joint['2'], joint['3'] = joint['3'], joint['2']
+ # return joint
+
+ ann_train_list = []
+ ann_test_list = []
+ img_train_list = []
+ img_test_list = []
+
+ def save_joints():
+ # joint_data_fn = os.path.join(path, 'data.json')
+ # fp = open(joint_data_fn, 'w')
+ mat = sio.loadmat(os.path.join(path, extracted_filename, "mpii_human_pose_v1_u12_1.mat"))
+
+ for _, (anno, train_flag) in enumerate( # all images
+ zip(mat['RELEASE']['annolist'][0, 0][0], mat['RELEASE']['img_train'][0, 0][0])):
+
+ img_fn = anno['image']['name'][0, 0][0]
+ train_flag = int(train_flag)
+
+ # print(i, img_fn, train_flag) # DEBUG print all images
+
+ if train_flag:
+ img_train_list.append(img_fn)
+ ann_train_list.append([])
+ else:
+ img_test_list.append(img_fn)
+ ann_test_list.append([])
+
+ head_rect = []
+ if 'x1' in str(anno['annorect'].dtype):
+ head_rect = zip(
+ [x1[0, 0] for x1 in anno['annorect']['x1'][0]], [y1[0, 0] for y1 in anno['annorect']['y1'][0]],
+ [x2[0, 0] for x2 in anno['annorect']['x2'][0]], [y2[0, 0] for y2 in anno['annorect']['y2'][0]]
+ )
+ else:
+ head_rect = [] # TODO
+
+ if 'annopoints' in str(anno['annorect'].dtype):
+ annopoints = anno['annorect']['annopoints'][0]
+ head_x1s = anno['annorect']['x1'][0]
+ head_y1s = anno['annorect']['y1'][0]
+ head_x2s = anno['annorect']['x2'][0]
+ head_y2s = anno['annorect']['y2'][0]
+
+ for annopoint, head_x1, head_y1, head_x2, head_y2 in zip(annopoints, head_x1s, head_y1s, head_x2s,
+ head_y2s):
+ # if annopoint != []:
+ # if len(annopoint) != 0:
+ if annopoint.size:
+ head_rect = [
+ float(head_x1[0, 0]),
+ float(head_y1[0, 0]),
+ float(head_x2[0, 0]),
+ float(head_y2[0, 0])
+ ]
+
+ # joint coordinates
+ annopoint = annopoint['point'][0, 0]
+ j_id = [str(j_i[0, 0]) for j_i in annopoint['id'][0]]
+ x = [x[0, 0] for x in annopoint['x'][0]]
+ y = [y[0, 0] for y in annopoint['y'][0]]
+ joint_pos = {}
+ for _j_id, (_x, _y) in zip(j_id, zip(x, y)):
+ joint_pos[int(_j_id)] = [float(_x), float(_y)]
+ # joint_pos = fix_wrong_joints(joint_pos)
+
+ # visibility list
+ if 'is_visible' in str(annopoint.dtype):
+ vis = [v[0] if v.size > 0 else [0] for v in annopoint['is_visible'][0]]
+ vis = dict([(k, int(v[0])) if len(v) > 0 else v for k, v in zip(j_id, vis)])
+ else:
+ vis = None
+
+ # if len(joint_pos) == 16:
+ if ((is_16_pos_only ==True) and (len(joint_pos) == 16)) or (is_16_pos_only == False):
+ # only use image with 16 key points / or use all
+ data = {
+ 'filename': img_fn,
+ 'train': train_flag,
+ 'head_rect': head_rect,
+ 'is_visible': vis,
+ 'joint_pos': joint_pos
+ }
+ # print(json.dumps(data), file=fp) # py3
+ if train_flag:
+ ann_train_list[-1].append(data)
+ else:
+ ann_test_list[-1].append(data)
+
+ # def write_line(datum, fp):
+ # joints = sorted([[int(k), v] for k, v in datum['joint_pos'].items()])
+ # joints = np.array([j for i, j in joints]).flatten()
+ #
+ # out = [datum['filename']]
+ # out.extend(joints)
+ # out = [str(o) for o in out]
+ # out = ','.join(out)
+ #
+ # print(out, file=fp)
+
+ # def split_train_test():
+ # # fp_test = open('data/mpii/test_joints.csv', 'w')
+ # fp_test = open(os.path.join(path, 'test_joints.csv'), 'w')
+ # # fp_train = open('data/mpii/train_joints.csv', 'w')
+ # fp_train = open(os.path.join(path, 'train_joints.csv'), 'w')
+ # # all_data = open('data/mpii/data.json').readlines()
+ # all_data = open(os.path.join(path, 'data.json')).readlines()
+ # N = len(all_data)
+ # N_test = int(N * 0.1)
+ # N_train = N - N_test
+ #
+ # print('N:{}'.format(N))
+ # print('N_train:{}'.format(N_train))
+ # print('N_test:{}'.format(N_test))
+ #
+ # np.random.seed(1701)
+ # perm = np.random.permutation(N)
+ # test_indices = perm[:N_test]
+ # train_indices = perm[N_test:]
+ #
+ # print('train_indices:{}'.format(len(train_indices)))
+ # print('test_indices:{}'.format(len(test_indices)))
+ #
+ # for i in train_indices:
+ # datum = json.loads(all_data[i].strip())
+ # write_line(datum, fp_train)
+ #
+ # for i in test_indices:
+ # datum = json.loads(all_data[i].strip())
+ # write_line(datum, fp_test)
+
+ save_joints()
+ # split_train_test() #
+
+ # read images dir
+ logging.info("reading images list ...")
+ img_dir = os.path.join(path, extracted_filename2)
+ _img_list = load_file_list(path=os.path.join(path, extracted_filename2), regx='\\.jpg', printable=False)
+ # ann_list = json.load(open(os.path.join(path, 'data.json')))
+ for i, im in enumerate(img_train_list):
+ if im not in _img_list:
+ print('missing training image {} in {} (remove from img(ann)_train_list)'.format(im, img_dir))
+ # img_train_list.remove(im)
+ del img_train_list[i]
+ del ann_train_list[i]
+ for i, im in enumerate(img_test_list):
+ if im not in _img_list:
+ print('missing testing image {} in {} (remove from img(ann)_test_list)'.format(im, img_dir))
+ # img_test_list.remove(im)
+ del img_train_list[i]
+ del ann_train_list[i]
+
+ # check annotation and images
+ n_train_images = len(img_train_list)
+ n_test_images = len(img_test_list)
+ n_images = n_train_images + n_test_images
+ logging.info("n_images: {} n_train_images: {} n_test_images: {}".format(n_images, n_train_images, n_test_images))
+ n_train_ann = len(ann_train_list)
+ n_test_ann = len(ann_test_list)
+ n_ann = n_train_ann + n_test_ann
+ logging.info("n_ann: {} n_train_ann: {} n_test_ann: {}".format(n_ann, n_train_ann, n_test_ann))
+ n_train_people = len(sum(ann_train_list, []))
+ n_test_people = len(sum(ann_test_list, []))
+ n_people = n_train_people + n_test_people
+ logging.info("n_people: {} n_train_people: {} n_test_people: {}".format(n_people, n_train_people, n_test_people))
+ # add path to all image file name
+ for i, value in enumerate(img_train_list):
+ img_train_list[i] = os.path.join(img_dir, value)
+ for i, value in enumerate(img_test_list):
+ img_test_list[i] = os.path.join(img_dir, value)
+ return img_train_list, ann_train_list, img_test_list, ann_test_list
+
+
+def save_npz(save_list=None, name='model.npz'):
+ """Input parameters and the file name, save parameters into .npz file. Use tl.utils.load_npz() to restore.
+
+ Parameters
+ ----------
+ save_list : list of tensor
+ A list of parameters (tensor) to be saved.
+ name : str
+ The name of the `.npz` file.
+
+ Examples
+ --------
+ Save model to npz
+
+ >>> tl.files.save_npz(network.all_weights, name='model.npz')
+
+ Load model from npz (Method 1)
+
+ >>> load_params = tl.files.load_npz(name='model.npz')
+ >>> tl.files.assign_weights(load_params, network)
+
+ Load model from npz (Method 2)
+
+ >>> tl.files.load_and_assign_npz(name='model.npz', network=network)
+
+ References
+ ----------
+ `Saving dictionary using numpy `__
+
+ """
+ logging.info("[*] Saving TL weights into %s" % name)
+ if save_list is None:
+ save_list = []
+
+ if tl.BACKEND == 'tensorflow':
+ save_list_var = tf_variables_to_numpy(save_list)
+ elif tl.BACKEND == 'mindspore':
+ save_list_var = ms_variables_to_numpy(save_list)
+ else:
+ raise NotImplementedError("This backend is not supported")
+ # print(name, save_list_var)
+ np.savez(name, params=save_list_var)
+ save_list_var = None
+ del save_list_var
+ logging.info("[*] Saved")
+
+
+def load_npz(path='', name='model.npz'):
+ """Load the parameters of a Model saved by tl.files.save_npz().
+
+ Parameters
+ ----------
+ path : str
+ Folder path to `.npz` file.
+ name : str
+ The name of the `.npz` file.
+
+ Returns
+ --------
+ list of array
+ A list of parameters in order.
+
+ Examples
+ --------
+ - See ``tl.files.save_npz``
+
+ References
+ ----------
+ - `Saving dictionary using numpy `__
+
+ """
+ d = np.load(os.path.join(path, name), allow_pickle=True)
+ return d['params']
+
+
+def assign_params(**kwargs):
+ raise Exception("please change assign_params --> assign_weights")
+
+
+def assign_weights(weights, network):
+ """Assign the given parameters to the TensorLayer network.
+
+ Parameters
+ ----------
+ weights : list of array
+ A list of model weights (array) in order.
+ network : :class:`Layer`
+ The network to be assigned.
+
+ Returns
+ --------
+ 1) list of operations if in graph mode
+ A list of tf ops in order that assign weights. Support sess.run(ops) manually.
+ 2) list of tf variables if in eager mode
+ A list of tf variables (assigned weights) in order.
+
+ Examples
+ --------
+
+ References
+ ----------
+ - `Assign value to a TensorFlow variable `__
+
+ """
+ ops = []
+ if tl.BACKEND == 'tensorflow':
+ for idx, param in enumerate(weights):
+ ops.append(network.all_weights[idx].assign(param))
+
+ elif tl.BACKEND == 'mindspore':
+
+ class Assign_net(Cell):
+
+ def __init__(self, y):
+ super(Assign_net, self).__init__()
+ self.y = y
+
+ def construct(self, x):
+ Assign()(self.y, x)
+
+ for idx, param in enumerate(weights):
+ assign_param = Tensor(param, dtype=ms.float32)
+ # net = Assign_net(network.all_weights[idx])
+ # net(assign_param)
+ Assign()(network.all_weights[idx], assign_param)
+ return ops
+
+
+def load_and_assign_npz(name=None, network=None):
+ """Load model from npz and assign to a network.
+
+ Parameters
+ -------------
+ name : str
+ The name of the `.npz` file.
+ network : :class:`Model`
+ The network to be assigned.
+
+ Examples
+ --------
+ - See ``tl.files.save_npz``
+
+ """
+ if network is None:
+ raise ValueError("network is None.")
+
+ if not os.path.exists(name):
+ logging.error("file {} doesn't exist.".format(name))
+ return False
+ else:
+ weights = load_npz(name=name)
+ assign_weights(weights, network)
+ logging.info("[*] Load {} SUCCESS!".format(name))
+
+
+def save_npz_dict(save_list=None, name='model.npz'):
+ """Input parameters and the file name, save parameters as a dictionary into .npz file.
+
+ Use ``tl.files.load_and_assign_npz_dict()`` to restore.
+
+ Parameters
+ ----------
+ save_list : list of parameters
+ A list of parameters (tensor) to be saved.
+ name : str
+ The name of the `.npz` file.
+
+ """
+ if save_list is None:
+ save_list = []
+
+ save_list_names = [tensor.name for tensor in save_list]
+ if tl.BACKEND == 'tensorflow':
+ save_list_var = tf_variables_to_numpy(save_list)
+ elif tl.BACKEND == 'mindspore':
+ save_list_var = ms_variables_to_numpy(save_list)
+ else:
+ raise NotImplementedError('Not implemented')
+ save_var_dict = {save_list_names[idx]: val for idx, val in enumerate(save_list_var)}
+ np.savez(name, **save_var_dict)
+ save_list_var = None
+ save_var_dict = None
+ del save_list_var
+ del save_var_dict
+ logging.info("[*] Model saved in npz_dict %s" % name)
+
+
+def load_and_assign_npz_dict(name='model.npz', network=None, skip=False):
+ """Restore the parameters saved by ``tl.files.save_npz_dict()``.
+
+ Parameters
+ -------------
+ name : str
+ The name of the `.npz` file.
+ network : :class:`Model`
+ The network to be assigned.
+ skip : boolean
+ If 'skip' == True, loaded weights whose name is not found in network's weights will be skipped.
+ If 'skip' is False, error will be raised when mismatch is found. Default False.
+
+ """
+ if not os.path.exists(name):
+ logging.error("file {} doesn't exist.".format(name))
+ return False
+
+ weights = np.load(name, allow_pickle=True)
+ if len(weights.keys()) != len(set(weights.keys())):
+ raise Exception("Duplication in model npz_dict %s" % name)
+
+ net_weights_name = [w.name for w in network.all_weights]
+
+ for key in weights.keys():
+ if key not in net_weights_name:
+ if skip:
+ logging.warning("Weights named '%s' not found in network. Skip it." % key)
+ else:
+ raise RuntimeError(
+ "Weights named '%s' not found in network. Hint: set argument skip=Ture "
+ "if you want to skip redundant or mismatch weights." % key
+ )
+ else:
+ if tl.BACKEND == 'tensorflow':
+ assign_tf_variable(network.all_weights[net_weights_name.index(key)], weights[key])
+ elif tl.BACKEND == 'mindspore':
+ assign_param = Tensor(weights[key], dtype=ms.float32)
+ assign_ms_variable(network.all_weights[net_weights_name.index(key)], assign_param)
+ logging.info("[*] Model restored from npz_dict %s" % name)
+
+
+def save_ckpt(mode_name='model.ckpt', save_dir='checkpoint', var_list=None, global_step=None, printable=False):
+ """Save parameters into `ckpt` file.
+
+ Parameters
+ ------------
+ mode_name : str
+ The name of the model, default is ``model.ckpt``.
+ save_dir : str
+ The path / file directory to the `ckpt`, default is ``checkpoint``.
+ var_list : list of tensor
+ The parameters / variables (tensor) to be saved. If empty, save all global variables (default).
+ global_step : int or None
+ Step number.
+ printable : boolean
+ Whether to print all parameters information.
+
+ See Also
+ --------
+ load_ckpt
+
+ """
+
+ if var_list is None:
+ if sess is None:
+ # FIXME: not sure whether global variables can be accessed in eager mode
+ raise ValueError(
+ "If var_list is None, sess must be specified. "
+ "In eager mode, can not access global variables easily. "
+ )
+ var_list = []
+
+ ckpt_file = os.path.join(save_dir, mode_name)
+ if var_list == []:
+ var_list = tf.global_variables()
+
+ logging.info("[*] save %s n_weights: %d" % (ckpt_file, len(var_list)))
+
+ if printable:
+ for idx, v in enumerate(var_list):
+ logging.info(" param {:3}: {:15} {}".format(idx, v.name, str(v.get_shape())))
+
+ if sess:
+ # graph mode
+ saver = tf.train.Saver(var_list)
+ saver.save(sess, ckpt_file, global_step=global_step)
+ else:
+ # eager mode
+ # saver = tfes.Saver(var_list)
+ # saver.save(ckpt_file, global_step=global_step)
+ # TODO: tf2.0 not stable, cannot import tensorflow.contrib.eager.python.saver
+ pass
+
+
+def load_ckpt(sess=None, mode_name='model.ckpt', save_dir='checkpoint', var_list=None, is_latest=True, printable=False):
+ """Load parameters from `ckpt` file.
+
+ Parameters
+ ------------
+ sess : Session
+ TensorFlow Session.
+ mode_name : str
+ The name of the model, default is ``model.ckpt``.
+ save_dir : str
+ The path / file directory to the `ckpt`, default is ``checkpoint``.
+ var_list : list of tensor
+ The parameters / variables (tensor) to be saved. If empty, save all global variables (default).
+ is_latest : boolean
+ Whether to load the latest `ckpt`, if False, load the `ckpt` with the name of ```mode_name``.
+ printable : boolean
+ Whether to print all parameters information.
+
+ Examples
+ ----------
+ - Save all global parameters.
+
+ >>> tl.files.save_ckpt(sess=sess, mode_name='model.ckpt', save_dir='model', printable=True)
+
+ - Save specific parameters.
+
+ >>> tl.files.save_ckpt(sess=sess, mode_name='model.ckpt', var_list=net.all_params, save_dir='model', printable=True)
+
+ - Load latest ckpt.
+
+ >>> tl.files.load_ckpt(sess=sess, var_list=net.all_params, save_dir='model', printable=True)
+
+ - Load specific ckpt.
+
+ >>> tl.files.load_ckpt(sess=sess, mode_name='model.ckpt', var_list=net.all_params, save_dir='model', is_latest=False, printable=True)
+
+ """
+ # if sess is None:
+ # raise ValueError("session is None.")
+ if var_list is None:
+ if sess is None:
+ # FIXME: not sure whether global variables can be accessed in eager mode
+ raise ValueError(
+ "If var_list is None, sess must be specified. "
+ "In eager mode, can not access global variables easily. "
+ )
+ var_list = []
+
+ if is_latest:
+ ckpt_file = tf.train.latest_checkpoint(save_dir)
+ else:
+ ckpt_file = os.path.join(save_dir, mode_name)
+
+ if not var_list:
+ var_list = tf.global_variables()
+
+ logging.info("[*] load %s n_weights: %d" % (ckpt_file, len(var_list)))
+
+ if printable:
+ for idx, v in enumerate(var_list):
+ logging.info(" weights {:3}: {:15} {}".format(idx, v.name, str(v.get_shape())))
+
+ try:
+ if sess:
+ # graph mode
+ saver = tf.train.Saver(var_list)
+ saver.restore(sess, ckpt_file)
+ else:
+ # eager mode
+ # saver = tfes.Saver(var_list)
+ # saver.restore(ckpt_file)
+ # TODO: tf2.0 not stable, cannot import tensorflow.contrib.eager.python.saver
+ pass
+
+ except Exception as e:
+ logging.info(e)
+ logging.info("[*] load ckpt fail ...")
+
+
+def save_any_to_npy(save_dict=None, name='file.npy'):
+ """Save variables to `.npy` file.
+
+ Parameters
+ ------------
+ save_dict : directory
+ The variables to be saved.
+ name : str
+ File name.
+
+ Examples
+ ---------
+ >>> tl.files.save_any_to_npy(save_dict={'data': ['a','b']}, name='test.npy')
+ >>> data = tl.files.load_npy_to_any(name='test.npy')
+ >>> print(data)
+ {'data': ['a','b']}
+
+ """
+ if save_dict is None:
+ save_dict = {}
+ np.save(name, save_dict)
+
+
+def load_npy_to_any(path='', name='file.npy'):
+ """Load `.npy` file.
+
+ Parameters
+ ------------
+ path : str
+ Path to the file (optional).
+ name : str
+ File name.
+
+ Examples
+ ---------
+ - see tl.files.save_any_to_npy()
+
+ """
+ file_path = os.path.join(path, name)
+ try:
+ return np.load(file_path, allow_pickle=True).item()
+ except Exception:
+ return np.load(file_path, allow_pickle=True)
+ raise Exception("[!] Fail to load %s" % file_path)
+
+
+def file_exists(filepath):
+ """Check whether a file exists by given file path."""
+ return os.path.isfile(filepath)
+
+
+def folder_exists(folderpath):
+ """Check whether a folder exists by given folder path."""
+ return os.path.isdir(folderpath)
+
+
+def del_file(filepath):
+ """Delete a file by given file path."""
+ os.remove(filepath)
+
+
+def del_folder(folderpath):
+ """Delete a folder by given folder path."""
+ shutil.rmtree(folderpath)
+
+
+def read_file(filepath):
+ """Read a file and return a string.
+
+ Examples
+ ---------
+ >>> data = tl.files.read_file('data.txt')
+
+ """
+ with open(filepath, 'r') as afile:
+ return afile.read()
+
+
+def load_file_list(path=None, regx='\.jpg', printable=True, keep_prefix=False):
+ r"""Return a file list in a folder by given a path and regular expression.
+
+ Parameters
+ ----------
+ path : str or None
+ A folder path, if `None`, use the current directory.
+ regx : str
+ The regx of file name.
+ printable : boolean
+ Whether to print the files infomation.
+ keep_prefix : boolean
+ Whether to keep path in the file name.
+
+ Examples
+ ----------
+ >>> file_list = tl.files.load_file_list(path=None, regx='w1pre_[0-9]+\.(npz)')
+
+ """
+ if path is None:
+ path = os.getcwd()
+ file_list = os.listdir(path)
+ return_list = []
+ for _, f in enumerate(file_list):
+ if re.search(regx, f):
+ return_list.append(f)
+ # return_list.sort()
+ if keep_prefix:
+ for i, f in enumerate(return_list):
+ return_list[i] = os.path.join(path, f)
+
+ if printable:
+ logging.info('Match file list = %s' % return_list)
+ logging.info('Number of files = %d' % len(return_list))
+ return return_list
+
+
+def load_folder_list(path=""):
+ """Return a folder list in a folder by given a folder path.
+
+ Parameters
+ ----------
+ path : str
+ A folder path.
+
+ """
+ return [os.path.join(path, o) for o in os.listdir(path) if os.path.isdir(os.path.join(path, o))]
+
+
+def exists_or_mkdir(path, verbose=True):
+ """Check a folder by given name, if not exist, create the folder and return False,
+ if directory exists, return True.
+
+ Parameters
+ ----------
+ path : str
+ A folder path.
+ verbose : boolean
+ If True (default), prints results.
+
+ Returns
+ --------
+ boolean
+ True if folder already exist, otherwise, returns False and create the folder.
+
+ Examples
+ --------
+ >>> tl.files.exists_or_mkdir("checkpoints/train")
+
+ """
+ if not os.path.exists(path):
+ if verbose:
+ logging.info("[*] creates %s ..." % path)
+ os.makedirs(path)
+ return False
+ else:
+ if verbose:
+ logging.info("[!] %s exists ..." % path)
+ return True
+
+
+def maybe_download_and_extract(filename, working_directory, url_source, extract=False, expected_bytes=None):
+ """Checks if file exists in working_directory otherwise tries to dowload the file,
+ and optionally also tries to extract the file if format is ".zip" or ".tar"
+
+ Parameters
+ -----------
+ filename : str
+ The name of the (to be) dowloaded file.
+ working_directory : str
+ A folder path to search for the file in and dowload the file to
+ url : str
+ The URL to download the file from
+ extract : boolean
+ If True, tries to uncompress the dowloaded file is ".tar.gz/.tar.bz2" or ".zip" file, default is False.
+ expected_bytes : int or None
+ If set tries to verify that the downloaded file is of the specified size, otherwise raises an Exception, defaults is None which corresponds to no check being performed.
+
+ Returns
+ ----------
+ str
+ File path of the dowloaded (uncompressed) file.
+
+ Examples
+ --------
+ >>> down_file = tl.files.maybe_download_and_extract(filename='train-images-idx3-ubyte.gz',
+ ... working_directory='data/',
+ ... url_source='http://yann.lecun.com/exdb/mnist/')
+ >>> tl.files.maybe_download_and_extract(filename='ADEChallengeData2016.zip',
+ ... working_directory='data/',
+ ... url_source='http://sceneparsing.csail.mit.edu/data/',
+ ... extract=True)
+
+ """
+
+ # We first define a download function, supporting both Python 2 and 3.
+ def _download(filename, working_directory, url_source):
+
+ progress_bar = progressbar.ProgressBar()
+
+ def _dlProgress(count, blockSize, totalSize, pbar=progress_bar):
+ if (totalSize != 0):
+
+ if not pbar.max_value:
+ totalBlocks = math.ceil(float(totalSize) / float(blockSize))
+ pbar.max_value = int(totalBlocks)
+
+ pbar.update(count, force=True)
+
+ filepath = os.path.join(working_directory, filename)
+
+ logging.info('Downloading %s...\n' % filename)
+
+ urlretrieve(url_source + filename, filepath, reporthook=_dlProgress)
+
+ exists_or_mkdir(working_directory, verbose=False)
+ filepath = os.path.join(working_directory, filename)
+
+ if not os.path.exists(filepath):
+
+ _download(filename, working_directory, url_source)
+ statinfo = os.stat(filepath)
+ logging.info('Succesfully downloaded %s %s bytes.' % (filename, statinfo.st_size)) # , 'bytes.')
+ if (not (expected_bytes is None) and (expected_bytes != statinfo.st_size)):
+ raise Exception('Failed to verify ' + filename + '. Can you get to it with a browser?')
+ if (extract):
+ if tarfile.is_tarfile(filepath):
+ logging.info('Trying to extract tar file')
+ tarfile.open(filepath, 'r').extractall(working_directory)
+ logging.info('... Success!')
+ elif zipfile.is_zipfile(filepath):
+ logging.info('Trying to extract zip file')
+ with zipfile.ZipFile(filepath) as zf:
+ zf.extractall(working_directory)
+ logging.info('... Success!')
+ else:
+ logging.info("Unknown compression_format only .tar.gz/.tar.bz2/.tar and .zip supported")
+ return filepath
+
+
+def natural_keys(text):
+ """Sort list of string with number in human order.
+
+ Examples
+ ----------
+ >>> l = ['im1.jpg', 'im31.jpg', 'im11.jpg', 'im21.jpg', 'im03.jpg', 'im05.jpg']
+ >>> l.sort(key=tl.files.natural_keys)
+ ['im1.jpg', 'im03.jpg', 'im05', 'im11.jpg', 'im21.jpg', 'im31.jpg']
+ >>> l.sort() # that is what we dont want
+ ['im03.jpg', 'im05', 'im1.jpg', 'im11.jpg', 'im21.jpg', 'im31.jpg']
+
+ References
+ ----------
+ - `link `__
+
+ """
+
+ # - alist.sort(key=natural_keys) sorts in human order
+ # http://nedbatchelder.com/blog/200712/human_sorting.html
+ # (See Toothy's implementation in the comments)
+ def atoi(text):
+ return int(text) if text.isdigit() else text
+
+ return [atoi(c) for c in re.split('(\d+)', text)]
+
+
+# Visualizing npz files
+def npz_to_W_pdf(path=None, regx='w1pre_[0-9]+\.(npz)'):
+ r"""Convert the first weight matrix of `.npz` file to `.pdf` by using `tl.visualize.W()`.
+
+ Parameters
+ ----------
+ path : str
+ A folder path to `npz` files.
+ regx : str
+ Regx for the file name.
+
+ Examples
+ ---------
+ Convert the first weight matrix of w1_pre...npz file to w1_pre...pdf.
+
+ >>> tl.files.npz_to_W_pdf(path='/Users/.../npz_file/', regx='w1pre_[0-9]+\.(npz)')
+
+ """
+ file_list = load_file_list(path=path, regx=regx)
+ for f in file_list:
+ W = load_npz(path, f)[0]
+ logging.info("%s --> %s" % (f, f.split('.')[0] + '.pdf'))
+ visualize.draw_weights(W, second=10, saveable=True, name=f.split('.')[0], fig_idx=2012)
+
+
+def tf_variables_to_numpy(variables):
+ """Convert TF tensor or a list of tensors into a list of numpy array"""
+ if not isinstance(variables, list):
+ var_list = [variables]
+ else:
+ var_list = variables
+
+ results = [v.numpy() for v in var_list]
+ return results
+
+
+def ms_variables_to_numpy(variables):
+ """Convert MS tensor or list of tensors into a list of numpy array"""
+ if not isinstance(variables, list):
+ var_list = [variables]
+ else:
+ var_list = variables
+
+ results = [v.data.asnumpy() for v in var_list]
+ return results
+
+
+def assign_tf_variable(variable, value):
+ """Assign value to a TF variable"""
+ variable.assign(value)
+
+
+def assign_ms_variable(variable, value):
+
+ class Assign_net(Cell):
+
+ def __init__(self, y):
+ super(Assign_net, self).__init__()
+ self.y = y
+
+ def construct(self, x):
+ Assign()(self.y, x)
+
+ # net = Assign_net(variable)
+ # net(value)
+ Assign()(variable, value)
+
+
+def _save_weights_to_hdf5_group(f, layers):
+ """
+ Save layer/model weights into hdf5 group recursively.
+
+ Parameters
+ ----------
+ f: hdf5 group
+ A hdf5 group created by h5py.File() or create_group().
+ layers: list
+ A list of layers to save weights.
+
+ """
+ f.attrs['layer_names'] = [layer.name.encode('utf8') for layer in layers]
+
+ for layer in layers:
+ g = f.create_group(layer.name)
+ if isinstance(layer, tl.models.Model):
+ _save_weights_to_hdf5_group(g, layer.all_layers)
+ elif isinstance(layer, tl.layers.ModelLayer):
+ _save_weights_to_hdf5_group(g, layer.model.all_layers)
+ elif isinstance(layer, tl.layers.LayerList):
+ _save_weights_to_hdf5_group(g, layer.layers)
+ elif isinstance(layer, tl.layers.Layer):
+ if layer.all_weights is not None:
+ weight_values = tf_variables_to_numpy(layer.all_weights)
+ weight_names = [w.name.encode('utf8') for w in layer.all_weights]
+ else:
+ weight_values = []
+ weight_names = []
+ g.attrs['weight_names'] = weight_names
+ for name, val in zip(weight_names, weight_values):
+ val_dataset = g.create_dataset(name, val.shape, dtype=val.dtype)
+ if not val.shape:
+ # scalar
+ val_dataset[()] = val
+ else:
+ val_dataset[:] = val
+ else:
+ raise Exception("Only layer or model can be saved into hdf5.")
+
+
+def _load_weights_from_hdf5_group_in_order(f, layers):
+ """
+ Load layer weights from a hdf5 group sequentially.
+
+ Parameters
+ ----------
+ f: hdf5 group
+ A hdf5 group created by h5py.File() or create_group().
+ layers: list
+ A list of layers to load weights.
+
+ """
+ layer_names = [n.decode('utf8') for n in f.attrs["layer_names"]]
+
+ for idx, name in enumerate(layer_names):
+ g = f[name]
+ layer = layers[idx]
+ if isinstance(layer, tl.models.Model):
+ _load_weights_from_hdf5_group_in_order(g, layer.all_layers)
+ elif isinstance(layer, tl.layers.ModelLayer):
+ _load_weights_from_hdf5_group_in_order(g, layer.model.all_layers)
+ elif isinstance(layer, tl.layers.LayerList):
+ _load_weights_from_hdf5_group_in_order(g, layer.layers)
+ elif isinstance(layer, tl.layers.Layer):
+ weight_names = [n.decode('utf8') for n in g.attrs['weight_names']]
+ for iid, w_name in enumerate(weight_names):
+ assign_tf_variable(layer.all_weights[iid], np.asarray(g[w_name]))
+ else:
+ raise Exception("Only layer or model can be saved into hdf5.")
+ if idx == len(layers) - 1:
+ break
+
+
+def _load_weights_from_hdf5_group(f, layers, skip=False):
+ """
+ Load layer weights from a hdf5 group by layer name.
+
+ Parameters
+ ----------
+ f: hdf5 group
+ A hdf5 group created by h5py.File() or create_group().
+ layers: list
+ A list of layers to load weights.
+ skip : boolean
+ If 'skip' == True, loaded layer whose name is not found in 'layers' will be skipped. If 'skip' is False,
+ error will be raised when mismatch is found. Default False.
+
+ """
+ layer_names = [n.decode('utf8') for n in f.attrs["layer_names"]]
+ layer_index = {layer.name: layer for layer in layers}
+
+ for idx, name in enumerate(layer_names):
+ if name not in layer_index.keys():
+ if skip:
+ logging.warning("Layer named '%s' not found in network. Skip it." % name)
+ else:
+ raise RuntimeError(
+ "Layer named '%s' not found in network. Hint: set argument skip=Ture "
+ "if you want to skip redundant or mismatch Layers." % name
+ )
+ else:
+ g = f[name]
+ layer = layer_index[name]
+ if isinstance(layer, tl.models.Model):
+ _load_weights_from_hdf5_group(g, layer.all_layers, skip)
+ elif isinstance(layer, tl.layers.ModelLayer):
+ _load_weights_from_hdf5_group(g, layer.model.all_layers, skip)
+ elif isinstance(layer, tl.layers.LayerList):
+ _load_weights_from_hdf5_group(g, layer.layers, skip)
+ elif isinstance(layer, tl.layers.Layer):
+ weight_names = [n.decode('utf8') for n in g.attrs['weight_names']]
+ for iid, w_name in enumerate(weight_names):
+ # FIXME : this is only for compatibility
+ if isinstance(layer, tl.layers.BatchNorm) and np.asarray(g[w_name]).ndim > 1:
+ assign_tf_variable(layer.all_weights[iid], np.asarray(g[w_name]).squeeze())
+ continue
+ assign_tf_variable(layer.all_weights[iid], np.asarray(g[w_name]))
+ else:
+ raise Exception("Only layer or model can be saved into hdf5.")
+
+
+def save_weights_to_hdf5(filepath, network):
+ """Input filepath and save weights in hdf5 format.
+
+ Parameters
+ ----------
+ filepath : str
+ Filename to which the weights will be saved.
+ network : Model
+ TL model.
+
+ Returns
+ -------
+
+ """
+ logging.info("[*] Saving TL weights into %s" % filepath)
+
+ with h5py.File(filepath, 'w') as f:
+ _save_weights_to_hdf5_group(f, network.all_layers)
+
+ logging.info("[*] Saved")
+
+
+def load_hdf5_to_weights_in_order(filepath, network):
+ """Load weights sequentially from a given file of hdf5 format
+
+ Parameters
+ ----------
+ filepath : str
+ Filename to which the weights will be loaded, should be of hdf5 format.
+ network : Model
+ TL model.
+
+ Notes:
+ If the file contains more weights than given 'weights', then the redundant ones will be ignored
+ if all previous weights match perfectly.
+
+ Returns
+ -------
+
+ """
+ f = h5py.File(filepath, 'r')
+ try:
+ layer_names = [n.decode('utf8') for n in f.attrs["layer_names"]]
+ except Exception:
+ raise NameError(
+ "The loaded hdf5 file needs to have 'layer_names' as attributes. "
+ "Please check whether this hdf5 file is saved from TL."
+ )
+
+ if len(network.all_layers) != len(layer_names):
+ logging.warning(
+ "Number of weights mismatch."
+ "Trying to load a saved file with " + str(len(layer_names)) + " layers into a model with " +
+ str(len(network.all_layers)) + " layers."
+ )
+
+ _load_weights_from_hdf5_group_in_order(f, network.all_layers)
+
+ f.close()
+ logging.info("[*] Load %s SUCCESS!" % filepath)
+
+
+def load_hdf5_to_weights(filepath, network, skip=False):
+ """Load weights by name from a given file of hdf5 format
+
+ Parameters
+ ----------
+ filepath : str
+ Filename to which the weights will be loaded, should be of hdf5 format.
+ network : Model
+ TL model.
+ skip : bool
+ If 'skip' == True, loaded weights whose name is not found in 'weights' will be skipped. If 'skip' is False,
+ error will be raised when mismatch is found. Default False.
+
+ Returns
+ -------
+
+ """
+ f = h5py.File(filepath, 'r')
+ try:
+ layer_names = [n.decode('utf8') for n in f.attrs["layer_names"]]
+ except Exception:
+ raise NameError(
+ "The loaded hdf5 file needs to have 'layer_names' as attributes. "
+ "Please check whether this hdf5 file is saved from TL."
+ )
+
+ net_index = {layer.name: layer for layer in network.all_layers}
+
+ if len(network.all_layers) != len(layer_names):
+ logging.warning(
+ "Number of weights mismatch."
+ "Trying to load a saved file with " + str(len(layer_names)) + " layers into a model with " +
+ str(len(network.all_layers)) + " layers."
+ )
+
+ # check mismatch form network weights to hdf5
+ for name in net_index.keys():
+ if name not in layer_names:
+ logging.warning("Network layer named '%s' not found in loaded hdf5 file. It will be skipped." % name)
+
+ # load weights from hdf5 to network
+ _load_weights_from_hdf5_group(f, network.all_layers, skip)
+
+ f.close()
+ logging.info("[*] Load %s SUCCESS!" % filepath)
+
+
+def load_and_assign_ckpt(model_dir, network=None, skip=True):
+ """Load weights by name from a given file of ckpt format
+
+ Parameters
+ ----------
+ model_dir : str
+ Filename to which the weights will be loaded, should be of ckpt format.
+ Examples: model_dir = /root/cnn_model/
+ network : Model
+ TL model.
+ skip : bool
+ If 'skip' == True, loaded weights whose name is not found in 'weights' will be skipped. If 'skip' is False,
+ error will be raised when mismatch is found. Default False.
+
+ Returns
+ -------
+
+ """
+ model_dir = model_dir
+ model_path = None
+ for root, dirs, files in os.walk(model_dir):
+ for file in files:
+ filename, extension = os.path.splitext(file)
+ if extension in ['.data-00000-of-00001', '.index', '.meta']:
+ model_path = model_dir + '/' + filename
+ break
+ if model_path == None:
+ raise Exception('The ckpt file is not found')
+
+ reader = pywrap_tensorflow.NewCheckpointReader(model_path)
+ var_to_shape_map = reader.get_variable_to_shape_map()
+
+ net_weights_name = [w.name for w in network.all_weights]
+
+ for key in var_to_shape_map:
+ if key not in net_weights_name:
+ if skip:
+ logging.warning("Weights named '%s' not found in network. Skip it." % key)
+ else:
+ raise RuntimeError(
+ "Weights named '%s' not found in network. Hint: set argument skip=Ture "
+ "if you want to skip redundant or mismatch weights." % key
+ )
+ else:
+ assign_tf_variable(network.all_weights[net_weights_name.index(key)], reader.get_tensor(key))
+ logging.info("[*] Model restored from ckpt %s" % filename)
+
+
+def ckpt_to_npz_dict(model_dir, save_name='model.npz'):
+ """ Save ckpt weights to npz file
+
+ Parameters
+ ----------
+ model_dir : str
+ Filename to which the weights will be loaded, should be of ckpt format.
+ Examples: model_dir = /root/cnn_model/
+ save_name : str
+ The save_name of the `.npz` file.
+
+ Returns
+ -------
+
+ """
+ model_dir = model_dir
+ model_path = None
+ for root, dirs, files in os.walk(model_dir):
+ for file in files:
+ filename, extension = os.path.splitext(file)
+ if extension in ['.data-00000-of-00001', '.index', '.meta']:
+ model_path = model_dir + '/' + filename
+ break
+ if model_path == None:
+ raise Exception('The ckpt file is not found')
+
+ reader = pywrap_tensorflow.NewCheckpointReader(model_path)
+ var_to_shape_map = reader.get_variable_to_shape_map()
+
+ parameters_dict = {}
+ for key in sorted(var_to_shape_map):
+ parameters_dict[key] = reader.get_tensor(key)
+ np.savez(save_name, **parameters_dict)
+ parameters_dict = None
+ del parameters_dict
+ logging.info("[*] Ckpt weights saved in npz_dict %s" % save_name)
diff --git a/tensorlayer/initializers/__init__.py b/tensorlayer/initializers/__init__.py
new file mode 100644
index 0000000..80557bd
--- /dev/null
+++ b/tensorlayer/initializers/__init__.py
@@ -0,0 +1,25 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+# __all__ = [
+# 'Initializer', 'Zeros', 'Ones', 'Constant', 'RandomUniform', 'RandomNormal', 'TruncatedNormal',
+# 'deconv2d_bilinear_upsampling_initializer', 'He_Normal'
+# ]
+
+from .load_initializers_backend import Zeros
+from .load_initializers_backend import Ones
+from .load_initializers_backend import Constant
+from .load_initializers_backend import RandomUniform
+from .load_initializers_backend import RandomNormal
+from .load_initializers_backend import TruncatedNormal
+from .load_initializers_backend import deconv2d_bilinear_upsampling_initializer
+from .load_initializers_backend import HeNormal
+
+# Alias
+zeros = Zeros
+ones = Ones
+constant = Constant
+random_uniform = RandomUniform
+random_normal = RandomNormal
+truncated_normal = TruncatedNormal
+he_normal = HeNormal
\ No newline at end of file
diff --git a/tensorlayer/initializers/load_initializers_backend.py b/tensorlayer/initializers/load_initializers_backend.py
new file mode 100644
index 0000000..fc65bab
--- /dev/null
+++ b/tensorlayer/initializers/load_initializers_backend.py
@@ -0,0 +1,16 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, division, print_function
+from tensorlayer.backend.ops.load_backend import BACKEND
+
+if BACKEND == 'tensorflow':
+ from .tensorflow_initializers import *
+elif BACKEND == 'mindspore':
+ from .tensorflow_initializers import *
+elif BACKEND == 'dragon':
+ from .tensorflow_initializers import *
+elif BACKEND == 'paddle':
+ from .paddle_initializers import *
+else:
+ raise NotImplementedError("This backend is not supported")
diff --git a/tensorlayer/initializers/paddle_initializers.py b/tensorlayer/initializers/paddle_initializers.py
new file mode 100644
index 0000000..22ffa7a
--- /dev/null
+++ b/tensorlayer/initializers/paddle_initializers.py
@@ -0,0 +1,178 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from paddle.fluid.initializer import ConstantInitializer
+from paddle.fluid.initializer import UniformInitializer
+from paddle.fluid.initializer import NormalInitializer
+from paddle.fluid.initializer import TruncatedNormalInitializer
+from paddle.fluid.initializer import MSRAInitializer
+import paddle
+
+__all__ = [
+ 'Zeros', 'Ones', 'Constant', 'RandomUniform', 'RandomNormal', 'TruncatedNormal',
+ 'deconv2d_bilinear_upsampling_initializer', 'HeNormal'
+]
+
+
+class Zeros(ConstantInitializer):
+ """Initializer that generates tensors initialized to 0.
+ """
+
+ def __init__(self):
+ super(Zeros, self).__init__(value=0.0, force_cpu=False)
+
+
+class Ones(object):
+ """Initializer that generates tensors initialized to 1.
+ """
+
+ def __init__(self):
+ # super(Ones, self).__init__(value=1.0, force_cpu=False)
+ pass
+
+ def __call__(self, shape, dtype):
+ return paddle.ones(shape=shape, dtype=dtype)
+
+
+class Constant(ConstantInitializer):
+ """Initializer that generates tensors initialized to a constant value.
+
+ Parameters
+ ----------
+ value : A python scalar or a numpy array.
+ The assigned value.
+
+ """
+
+ def __init__(self, value=0.0):
+ if value is None:
+ raise ValueError("value must not be none.")
+ super(Constant, self).__init__(value=value, force_cpu=False)
+ self.value = value
+
+ def get_config(self):
+ return {"value": self.value}
+
+
+class RandomUniform(UniformInitializer):
+ """Initializer that generates tensors with a uniform distribution.
+
+ Parameters
+ ----------
+ minval : A python scalar or a scalar tensor.
+ Lower bound of the range of random values to generate.
+ maxval : A python scalar or a scalar tensor.
+ Upper bound of the range of random values to generate.
+ seed : A Python integer.
+ Used to seed the random generator.
+
+ """
+
+ def __init__(self, minval=-0.05, maxval=0.05, seed=0):
+ assert minval is not None, 'low should not be None'
+ assert maxval is not None, 'high should not be None'
+ assert maxval >= minval, 'high should greater or equal than low'
+ super(RandomUniform, self).__init__(
+ low=minval, high=maxval, seed=seed, diag_num=0, diag_step=0, diag_val=1.0)
+ self.minval = minval
+ self.maxval = maxval
+ self.seed = seed
+
+ def get_config(self):
+ return {"minval": self.minval, "maxval": self.maxval, "seed": self.seed}
+
+
+class RandomNormal(NormalInitializer):
+ """Initializer that generates tensors with a normal distribution.
+
+ Parameters
+ ----------
+ mean : A python scalar or a scalar tensor.
+ Mean of the random values to generate.
+ stddev : A python scalar or a scalar tensor.
+ Standard deviation of the random values to generate.
+ seed : A Python integer.
+ Used to seed the random generator.
+ """
+
+ def __init__(self, mean=0.0, stddev=0.05, seed=0):
+ assert mean is not None, 'mean should not be None'
+ assert stddev is not None, 'std should not be None'
+ super(RandomNormal, self).__init__(loc=mean, scale=stddev, seed=seed)
+ self.mean = mean
+ self.stddev = stddev
+ self.seed = seed
+
+ def get_config(self):
+ return {"mean": self.mean, "stddev": self.stddev, "seed": self.seed}
+
+
+class TruncatedNormal(TruncatedNormalInitializer):
+ """Initializer that generates a truncated normal distribution.
+
+ These values are similar to values from a `RandomNormal`
+ except that values more than two standard deviations from the mean
+ are discarded and re-drawn. This is the recommended initializer for
+ neural network weights and filters.
+
+
+ Parameters
+ ----------
+ mean : A python scalar or a scalar tensor.
+ Mean of the random values to generate.
+ stddev : A python scalar or a scalar tensor.
+ Standard deviation of the andom values to generate.
+ seed : A Python integer.
+ Used to seed the random generator.
+ """
+
+ def __init__(self, mean=0.0, stddev=0.05, seed=0):
+ assert mean is not None, 'mean should not be None'
+ assert stddev is not None, 'std should not be None'
+ super(TruncatedNormal, self).__init__(loc=mean, scale=stddev, seed=seed)
+ self.mean = mean
+ self.stddev = stddev
+ self.seed = seed
+
+ def get_config(self):
+ return {"mean": self.mean, "stddev": self.stddev, "seed": self.seed}
+
+
+class HeNormal(MSRAInitializer):
+ """He normal initializer.
+
+ Parameters
+ ----------
+ seed : A Python integer.
+ Used to seed the random generator.
+
+ """
+
+ def __init__(self, seed=0):
+ super(HeNormal, self).__init__(
+ uniform=False, fan_in=None, seed=seed)
+ self.seed = seed
+
+ def get_config(self):
+ return {"seed", self.seed}
+
+
+def deconv2d_bilinear_upsampling_initializer(shape):
+ """Returns the initializer that can be passed to DeConv2dLayer for initializing the
+ weights in correspondence to channel-wise bilinear up-sampling.
+ Used in segmentation approaches such as [FCN](https://arxiv.org/abs/1605.06211)
+
+ Parameters
+ ----------
+ shape : tuple of int
+ The shape of the filters, [height, width, output_channels, in_channels].
+ It must match the shape passed to DeConv2dLayer.
+
+ Returns
+ -------
+ ``tf.constant_initializer``
+ A constant initializer with weights set to correspond to per channel bilinear upsampling
+ when passed as W_int in DeConv2dLayer
+
+ """
+ raise NotImplementedError
diff --git a/tensorlayer/initializers/tensorflow_initializers.py b/tensorlayer/initializers/tensorflow_initializers.py
new file mode 100644
index 0000000..8865216
--- /dev/null
+++ b/tensorlayer/initializers/tensorflow_initializers.py
@@ -0,0 +1,243 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import tensorlayer as tl
+
+__all__ = [
+ 'Initializer', 'Zeros', 'Ones', 'Constant', 'RandomUniform', 'RandomNormal', 'TruncatedNormal',
+ 'deconv2d_bilinear_upsampling_initializer', 'HeNormal'
+]
+
+
+class Initializer(object):
+ """Initializer base class: all initializers inherit from this class.
+ """
+
+ def __call__(self, shape, dtype=None):
+ """Returns a tensor object initialized as specified by the initializer.
+
+ Parameters
+ ----------
+ shape : tuple of int.
+ The shape of the tensor.
+ dtype : Optional dtype of the tensor.
+ If not provided will return tensor of `tl.float32`.
+
+ Returns
+ -------
+
+ """
+ raise NotImplementedError
+
+ def get_config(self):
+ """Returns the configuration of the initializer as a JSON-serializable dict.
+
+ Returns
+ -------
+ A JSON-serializable Python dict.
+ """
+ return {}
+
+ @classmethod
+ def from_config(cls, config):
+ """Instantiates an initializer from a configuration dictionary.
+
+ Parameters
+ ----------
+ config : A python dictionary.
+ It will typically be the output of `get_config`.
+
+ Returns
+ -------
+ An Initializer instance.
+ """
+ if 'dtype' in config:
+ config.pop('dtype')
+ return cls(**config)
+
+
+class Zeros(Initializer):
+ """Initializer that generates tensors initialized to 0.
+ """
+
+ def __call__(self, shape, dtype=tl.float32):
+ return tl.zeros(shape, dtype=dtype)
+
+
+class Ones(Initializer):
+ """Initializer that generates tensors initialized to 1.
+ """
+
+ def __call__(self, shape, dtype=tl.float32):
+ return tl.ones(shape, dtype=dtype)
+
+
+class Constant(Initializer):
+ """Initializer that generates tensors initialized to a constant value.
+
+ Parameters
+ ----------
+ value : A python scalar or a numpy array.
+ The assigned value.
+
+ """
+
+ def __init__(self, value=0):
+ self.value = value
+
+ def __call__(self, shape, dtype=tl.float32):
+ return tl.constant(self.value, shape=shape, dtype=dtype)
+
+ def get_config(self):
+ return {"value": self.value}
+
+
+class RandomUniform(Initializer):
+ """Initializer that generates tensors with a uniform distribution.
+
+ Parameters
+ ----------
+ minval : A python scalar or a scalar tensor.
+ Lower bound of the range of random values to generate.
+ maxval : A python scalar or a scalar tensor.
+ Upper bound of the range of random values to generate.
+ seed : A Python integer.
+ Used to seed the random generator.
+
+ """
+
+ def __init__(self, minval=-0.05, maxval=0.05, seed=None):
+ self.minval = minval
+ self.maxval = maxval
+ self.seed = seed
+
+ def __call__(self, shape, dtype=tl.float32):
+ return tl.random_uniform(shape, self.minval, self.maxval, dtype=dtype, seed=self.seed)
+
+ def get_config(self):
+ return {"minval": self.minval, "maxval": self.maxval, "seed": self.seed}
+
+
+class RandomNormal(Initializer):
+ """Initializer that generates tensors with a normal distribution.
+
+ Parameters
+ ----------
+ mean : A python scalar or a scalar tensor.
+ Mean of the random values to generate.
+ stddev : A python scalar or a scalar tensor.
+ Standard deviation of the random values to generate.
+ seed : A Python integer.
+ Used to seed the random generator.
+ """
+
+ def __init__(self, mean=0.0, stddev=0.05, seed=None):
+ self.mean = mean
+ self.stddev = stddev
+ self.seed = seed
+
+ def __call__(self, shape, dtype=tl.float32):
+ return tl.random_normal(shape, self.mean, self.stddev, dtype=dtype, seed=self.seed)
+
+ def get_config(self):
+ return {"mean": self.mean, "stddev": self.stddev, "seed": self.seed}
+
+
+class TruncatedNormal(Initializer):
+ """Initializer that generates a truncated normal distribution.
+
+ These values are similar to values from a `RandomNormal`
+ except that values more than two standard deviations from the mean
+ are discarded and re-drawn. This is the recommended initializer for
+ neural network weights and filters.
+
+
+ Parameters
+ ----------
+ mean : A python scalar or a scalar tensor.
+ Mean of the random values to generate.
+ stddev : A python scalar or a scalar tensor.
+ Standard deviation of the andom values to generate.
+ seed : A Python integer.
+ Used to seed the random generator.
+ """
+
+ def __init__(self, mean=0.0, stddev=0.05, seed=None):
+ self.mean = mean
+ self.stddev = stddev
+ self.seed = seed
+
+ def __call__(self, shape, dtype=tl.float32):
+ return tl.truncated_normal(shape, self.mean, self.stddev, dtype=dtype, seed=self.seed)
+
+ def get_config(self):
+ return {"mean": self.mean, "stddev": self.stddev, "seed": self.seed}
+
+
+class HeNormal(Initializer):
+ """He normal initializer.
+
+ Parameters
+ ----------
+ seed : A Python integer.
+ Used to seed the random generator.
+
+ """
+
+ def __init__(self, seed=None):
+ self.seed = seed
+
+ def __call__(self, shape, dtype=tl.float32):
+ return tl.he_normal(seed=self.seed, shape=shape, dtype=dtype)
+
+ def get_config(self):
+ return {"seed", self.seed}
+
+
+def deconv2d_bilinear_upsampling_initializer(shape):
+ """Returns the initializer that can be passed to DeConv2dLayer for initializing the
+ weights in correspondence to channel-wise bilinear up-sampling.
+ Used in segmentation approaches such as [FCN](https://arxiv.org/abs/1605.06211)
+
+ Parameters
+ ----------
+ shape : tuple of int
+ The shape of the filters, [height, width, output_channels, in_channels].
+ It must match the shape passed to DeConv2dLayer.
+
+ Returns
+ -------
+ ``tf.constant_initializer``
+ A constant initializer with weights set to correspond to per channel bilinear upsampling
+ when passed as W_int in DeConv2dLayer
+
+ """
+ if shape[0] != shape[1]:
+ raise Exception('deconv2d_bilinear_upsampling_initializer only supports symmetrical filter sizes')
+
+ if shape[3] < shape[2]:
+ raise Exception(
+ 'deconv2d_bilinear_upsampling_initializer behaviour is not defined for num_in_channels < num_out_channels '
+ )
+
+ filter_size = shape[0]
+ num_out_channels = shape[2]
+ num_in_channels = shape[3]
+
+ # Create bilinear filter kernel as numpy array
+ bilinear_kernel = np.zeros([filter_size, filter_size], dtype=np.float32)
+ scale_factor = (filter_size + 1) // 2
+ if filter_size % 2 == 1:
+ center = scale_factor - 1
+ else:
+ center = scale_factor - 0.5
+ for x in range(filter_size):
+ for y in range(filter_size):
+ bilinear_kernel[x, y] = (1 - abs(x - center) / scale_factor) * (1 - abs(y - center) / scale_factor)
+ weights = np.zeros((filter_size, filter_size, num_out_channels, num_in_channels), dtype=np.float32)
+ for i in range(num_out_channels):
+ weights[:, :, i, i] = bilinear_kernel
+
+ # assign numpy array to constant_initalizer and pass to get_variable
+ return Constant(value=weights)
diff --git a/tensorlayer/iterate.py b/tensorlayer/iterate.py
new file mode 100644
index 0000000..804bbb3
--- /dev/null
+++ b/tensorlayer/iterate.py
@@ -0,0 +1,283 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import numpy as np
+from six.moves import xrange
+
+__all__ = [
+ 'minibatches',
+ 'seq_minibatches',
+ 'seq_minibatches2',
+ 'ptb_iterator',
+]
+
+
+def minibatches(inputs=None, targets=None, batch_size=None, allow_dynamic_batch_size=False, shuffle=False):
+ """Generate a generator that input a group of example in numpy.array and
+ their labels, return the examples and labels by the given batch size.
+
+ Parameters
+ ----------
+ inputs : numpy.array
+ The input features, every row is a example.
+ targets : numpy.array
+ The labels of inputs, every row is a example.
+ batch_size : int
+ The batch size.
+ allow_dynamic_batch_size: boolean
+ Allow the use of the last data batch in case the number of examples is not a multiple of batch_size, this may result in unexpected behaviour if other functions expect a fixed-sized batch-size.
+ shuffle : boolean
+ Indicating whether to use a shuffling queue, shuffle the dataset before return.
+
+ Examples
+ --------
+ >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])
+ >>> y = np.asarray([0,1,2,3,4,5])
+ >>> for batch in tl.iterate.minibatches(inputs=X, targets=y, batch_size=2, shuffle=False):
+ >>> print(batch)
+ ... (array([['a', 'a'], ['b', 'b']], dtype=' len(inputs):
+ if allow_dynamic_batch_size:
+ end_idx = len(inputs)
+ else:
+ break
+ if shuffle:
+ excerpt = indices[start_idx:end_idx]
+ else:
+ excerpt = slice(start_idx, end_idx)
+ if (isinstance(inputs, list) or isinstance(targets, list)) and (shuffle ==True):
+ # zsdonghao: for list indexing when shuffle==True
+ yield [inputs[i] for i in excerpt], [targets[i] for i in excerpt]
+ else:
+ yield inputs[excerpt], targets[excerpt]
+
+
+def seq_minibatches(inputs, targets, batch_size, seq_length, stride=1):
+ """Generate a generator that return a batch of sequence inputs and targets.
+ If `batch_size=100` and `seq_length=5`, one return will have 500 rows (examples).
+
+ Parameters
+ ----------
+ inputs : numpy.array
+ The input features, every row is a example.
+ targets : numpy.array
+ The labels of inputs, every element is a example.
+ batch_size : int
+ The batch size.
+ seq_length : int
+ The sequence length.
+ stride : int
+ The stride step, default is 1.
+
+ Examples
+ --------
+ Synced sequence input and output.
+
+ >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])
+ >>> y = np.asarray([0, 1, 2, 3, 4, 5])
+ >>> for batch in tl.iterate.seq_minibatches(inputs=X, targets=y, batch_size=2, seq_length=2, stride=1):
+ >>> print(batch)
+ ... (array([['a', 'a'], ['b', 'b'], ['b', 'b'], ['c', 'c']], dtype='>> return_last = True
+ >>> num_steps = 2
+ >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])
+ >>> Y = np.asarray([0,1,2,3,4,5])
+ >>> for batch in tl.iterate.seq_minibatches(inputs=X, targets=Y, batch_size=2, seq_length=num_steps, stride=1):
+ >>> x, y = batch
+ >>> if return_last:
+ >>> tmp_y = y.reshape((-1, num_steps) + y.shape[1:])
+ >>> y = tmp_y[:, -1]
+ >>> print(x, y)
+ ... [['a' 'a']
+ ... ['b' 'b']
+ ... ['b' 'b']
+ ... ['c' 'c']] [1 2]
+ ... [['c' 'c']
+ ... ['d' 'd']
+ ... ['d' 'd']
+ ... ['e' 'e']] [3 4]
+
+ """
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+
+ n_loads = (batch_size * stride) + (seq_length - stride)
+
+ for start_idx in range(0, len(inputs) - n_loads + 1, (batch_size * stride)):
+ seq_inputs = np.zeros((batch_size, seq_length) + inputs.shape[1:], dtype=inputs.dtype)
+ seq_targets = np.zeros((batch_size, seq_length) + targets.shape[1:], dtype=targets.dtype)
+ for b_idx in xrange(batch_size):
+ start_seq_idx = start_idx + (b_idx * stride)
+ end_seq_idx = start_seq_idx + seq_length
+ seq_inputs[b_idx] = inputs[start_seq_idx:end_seq_idx]
+ seq_targets[b_idx] = targets[start_seq_idx:end_seq_idx]
+ flatten_inputs = seq_inputs.reshape((-1, ) + inputs.shape[1:])
+ flatten_targets = seq_targets.reshape((-1, ) + targets.shape[1:])
+ yield flatten_inputs, flatten_targets
+
+
+def seq_minibatches2(inputs, targets, batch_size, num_steps):
+ """Generate a generator that iterates on two list of words. Yields (Returns) the source contexts and
+ the target context by the given batch_size and num_steps (sequence_length).
+ In TensorFlow's tutorial, this generates the `batch_size` pointers into the raw PTB data, and allows minibatch iteration along these pointers.
+
+ Parameters
+ ----------
+ inputs : list of data
+ The context in list format; note that context usually be represented by splitting by space, and then convert to unique word IDs.
+ targets : list of data
+ The context in list format; note that context usually be represented by splitting by space, and then convert to unique word IDs.
+ batch_size : int
+ The batch size.
+ num_steps : int
+ The number of unrolls. i.e. sequence length
+
+ Yields
+ ------
+ Pairs of the batched data, each a matrix of shape [batch_size, num_steps].
+
+ Raises
+ ------
+ ValueError : if batch_size or num_steps are too high.
+
+ Examples
+ --------
+ >>> X = [i for i in range(20)]
+ >>> Y = [i for i in range(20,40)]
+ >>> for batch in tl.iterate.seq_minibatches2(X, Y, batch_size=2, num_steps=3):
+ ... x, y = batch
+ ... print(x, y)
+ ...
+ ... [[ 0. 1. 2.]
+ ... [ 10. 11. 12.]]
+ ... [[ 20. 21. 22.]
+ ... [ 30. 31. 32.]]
+ ...
+ ... [[ 3. 4. 5.]
+ ... [ 13. 14. 15.]]
+ ... [[ 23. 24. 25.]
+ ... [ 33. 34. 35.]]
+ ...
+ ... [[ 6. 7. 8.]
+ ... [ 16. 17. 18.]]
+ ... [[ 26. 27. 28.]
+ ... [ 36. 37. 38.]]
+
+ Notes
+ -----
+ - Hint, if the input data are images, you can modify the source code `data = np.zeros([batch_size, batch_len)` to `data = np.zeros([batch_size, batch_len, inputs.shape[1], inputs.shape[2], inputs.shape[3]])`.
+ """
+ if len(inputs) != len(targets):
+ raise AssertionError("The length of inputs and targets should be equal")
+
+ data_len = len(inputs)
+ batch_len = data_len // batch_size
+ # data = np.zeros([batch_size, batch_len])
+ data = np.zeros((batch_size, batch_len) + inputs.shape[1:], dtype=inputs.dtype)
+ data2 = np.zeros([batch_size, batch_len])
+
+ for i in range(batch_size):
+ data[i] = inputs[batch_len * i:batch_len * (i + 1)]
+ data2[i] = targets[batch_len * i:batch_len * (i + 1)]
+
+ epoch_size = (batch_len - 1) // num_steps
+
+ if epoch_size == 0:
+ raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
+
+ for i in range(epoch_size):
+ x = data[:, i * num_steps:(i + 1) * num_steps]
+ x2 = data2[:, i * num_steps:(i + 1) * num_steps]
+ yield (x, x2)
+
+
+def ptb_iterator(raw_data, batch_size, num_steps):
+ """Generate a generator that iterates on a list of words, see `PTB example `__.
+ Yields the source contexts and the target context by the given batch_size and num_steps (sequence_length).
+
+ In TensorFlow's tutorial, this generates `batch_size` pointers into the raw
+ PTB data, and allows minibatch iteration along these pointers.
+
+ Parameters
+ ----------
+ raw_data : a list
+ the context in list format; note that context usually be
+ represented by splitting by space, and then convert to unique
+ word IDs.
+ batch_size : int
+ the batch size.
+ num_steps : int
+ the number of unrolls. i.e. sequence_length
+
+ Yields
+ ------
+ Pairs of the batched data, each a matrix of shape [batch_size, num_steps].
+ The second element of the tuple is the same data time-shifted to the
+ right by one.
+
+ Raises
+ ------
+ ValueError : if batch_size or num_steps are too high.
+
+ Examples
+ --------
+ >>> train_data = [i for i in range(20)]
+ >>> for batch in tl.iterate.ptb_iterator(train_data, batch_size=2, num_steps=3):
+ >>> x, y = batch
+ >>> print(x, y)
+ ... [[ 0 1 2] <---x 1st subset/ iteration
+ ... [10 11 12]]
+ ... [[ 1 2 3] <---y
+ ... [11 12 13]]
+ ...
+ ... [[ 3 4 5] <--- 1st batch input 2nd subset/ iteration
+ ... [13 14 15]] <--- 2nd batch input
+ ... [[ 4 5 6] <--- 1st batch target
+ ... [14 15 16]] <--- 2nd batch target
+ ...
+ ... [[ 6 7 8] 3rd subset/ iteration
+ ... [16 17 18]]
+ ... [[ 7 8 9]
+ ... [17 18 19]]
+ """
+ raw_data = np.array(raw_data, dtype=np.int32)
+
+ data_len = len(raw_data)
+ batch_len = data_len // batch_size
+ data = np.zeros([batch_size, batch_len], dtype=np.int32)
+ for i in range(batch_size):
+ data[i] = raw_data[batch_len * i:batch_len * (i + 1)]
+
+ epoch_size = (batch_len - 1) // num_steps
+
+ if epoch_size == 0:
+ raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
+
+ for i in range(epoch_size):
+ x = data[:, i * num_steps:(i + 1) * num_steps]
+ y = data[:, i * num_steps + 1:(i + 1) * num_steps + 1]
+ yield (x, y)
diff --git a/tensorlayer/layers/__init__.py b/tensorlayer/layers/__init__.py
new file mode 100644
index 0000000..309d586
--- /dev/null
+++ b/tensorlayer/layers/__init__.py
@@ -0,0 +1,26 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from .activation import *
+from .convolution import *
+from .core import *
+from .dense import *
+from .deprecated import *
+from .dropout import *
+from .embedding import *
+from .extend import *
+from .image_resampling import *
+from .inputs import *
+from .lambda_layers import *
+from .merge import *
+from .noise import *
+from .normalization import *
+from .padding import *
+from .pooling import *
+from .quantize import *
+# from .recurrent import *
+from .scale import *
+from .shape import *
+from .spatial_transformer import *
+from .stack import *
+from .utils import *
diff --git a/tensorlayer/layers/activation.py b/tensorlayer/layers/activation.py
new file mode 100644
index 0000000..c5a0de3
--- /dev/null
+++ b/tensorlayer/layers/activation.py
@@ -0,0 +1,605 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from tensorlayer import logging
+import tensorlayer as tl
+from tensorlayer.initializers import truncated_normal
+from tensorlayer.layers.core import Module
+
+__all__ = [
+ 'PRelu',
+ 'PRelu6',
+ 'PTRelu6',
+ 'LeakyReLU',
+ 'LeakyReLU6',
+ 'LeakyTwiceRelu6',
+ 'Ramp',
+ 'Swish',
+ 'HardTanh',
+ 'Mish'
+]
+
+
+class PRelu(Module):
+ """
+ The :class:`PRelu` class is Parametric Rectified Linear layer.
+ It follows f(x) = alpha * x for x < 0, f(x) = x for x >= 0,
+ where alpha is a learned array with the same shape as x.
+
+ Parameters
+ ----------
+ channel_shared : boolean
+ If True, single weight is shared by all channels.
+ in_channels: int
+ The number of channels of the previous layer.
+ If None, it will be automatically detected when the layer is forwarded for the first time.
+ a_init : initializer
+ The initializer for initializing the alpha(s).
+ name : None or str
+ A unique layer name.
+
+ Examples
+ -----------
+ >>> inputs = tl.layers.Input([10, 5])
+ >>> prelulayer = tl.layers.PRelu(channel_shared=True)
+
+ References
+ -----------
+ - `Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification `__
+ - `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010]
+ 
+ 
+